{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10922}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10368.605, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56102bb91b40] error while decoding MB 16 13, bytestream -5 [in#0/rtsp @ 0x56102bb5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56102bb5ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 140, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:05:33+09:00", "last_error_summary": "[h264 @ 0x56102bb91b40] error while decoding MB 16 13, bytestream -5 [in#0/rtsp @ 0x56102bb5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56102bb5ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-06-30T23:56:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 523.51, "last_started_at": "2026-06-30T23:56:49+09:00", "pid": 67939, "process": {"exists": false, "pid": 67939}, "restart_count": 139, "restart_count_per_hour": 5, "rtsp_timeout_count": 226, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 140}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.957, "sample_count": 2576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1310656, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:05:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T00:05:33+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-01T00:05:33+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T00:05:33.186210+09:00", "mtime_epoch": 1782831933.1862104, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102849}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:05:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1678.203, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.737, "pushed": 0.737, "read": 0.793}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.244, "count": 13, "max_ms": 19.165, "p95_ms": 19.165}, "gstreamer_push": {"avg_ms": 1.216, "count": 13, "max_ms": 1.299, "p95_ms": 1.299}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.896, "count": 13, "max_ms": 39.467, "p95_ms": 39.467}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 45.981, "count": 13, "max_ms": 47.041, "p95_ms": 47.041}, "yolo11_640_fire_11class": {"avg_ms": 47.229, "count": 13, "max_ms": 49.608, "p95_ms": 49.608}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.193, "count": 13, "max_ms": 47.757, "p95_ms": 47.757}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.13, "count": 13, "max_ms": 75.881, "p95_ms": 75.881}, "read_interval": {"avg_ms": 2266.021, "count": 14, "max_ms": 16530.515, "p95_ms": 16530.515}, "triton_wait": {"avg_ms": 47.818, "count": 13, "max_ms": 49.621, "p95_ms": 49.621}, "visualize": {"avg_ms": 3.429, "count": 13, "max_ms": 3.671, "p95_ms": 3.671}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.645, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10922}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10368.055, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:05:33+09:00", "last_error_summary": "[h264 @ 0x56102bb91b40] error while decoding MB 16 13, bytestream -5 [in#0/rtsp @ 0x56102bb5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56102bb5ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 140, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:05:33+09:00", "last_error_summary": "[h264 @ 0x56102bb91b40] error while decoding MB 16 13, bytestream -5 [in#0/rtsp @ 0x56102bb5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56102bb5ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:05:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 523.536, "last_started_at": "2026-06-30T23:56:49+09:00", "pid": 67939, "process": {"exists": false, "pid": 67939}, "restart_count": 139, "restart_count_per_hour": 5, "rtsp_timeout_count": 226, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 140}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.957, "sample_count": 2576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1310657, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:05:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T00:05:33+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-01T00:05:33+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T00:05:33.186210+09:00", "mtime_epoch": 1782831933.1862104, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102849}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:05:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1678.203, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.792, "pushed": 0.792, "read": 0.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.453, "count": 14, "max_ms": 21.165, "p95_ms": 21.165}, "gstreamer_push": {"avg_ms": 1.226, "count": 14, "max_ms": 1.358, "p95_ms": 1.358}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.934, "count": 14, "max_ms": 39.467, "p95_ms": 39.467}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.03, "count": 14, "max_ms": 47.041, "p95_ms": 47.041}, "yolo11_640_fire_11class": {"avg_ms": 47.188, "count": 14, "max_ms": 49.608, "p95_ms": 49.608}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.249, "count": 14, "max_ms": 47.974, "p95_ms": 47.974}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.565, "count": 14, "max_ms": 80.21, "p95_ms": 80.21}, "read_interval": {"avg_ms": 2266.021, "count": 14, "max_ms": 16530.515, "p95_ms": 16530.515}, "triton_wait": {"avg_ms": 47.83, "count": 14, "max_ms": 49.621, "p95_ms": 49.621}, "visualize": {"avg_ms": 3.619, "count": 14, "max_ms": 6.081, "p95_ms": 6.081}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.671, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10922}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10368.027, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 140, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:05:33+09:00", "last_error_summary": "[h264 @ 0x56102bb91b40] error while decoding MB 16 13, bytestream -5 [in#0/rtsp @ 0x56102bb5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56102bb5ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:05:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 523.543, "last_started_at": "2026-06-30T23:56:49+09:00", "pid": 67939, "process": {"exists": false, "pid": 67939}, "restart_count": 139, "restart_count_per_hour": 5, "rtsp_timeout_count": 226, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 140}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.957, "sample_count": 2576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1310657, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:05:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-01T00:05:33+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-01T00:05:33+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-01T00:05:33.186210+09:00", "mtime_epoch": 1782831933.1862104, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102849}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:05:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1678.203, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.792, "pushed": 0.792, "read": 0.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.453, "count": 14, "max_ms": 21.165, "p95_ms": 21.165}, "gstreamer_push": {"avg_ms": 1.226, "count": 14, "max_ms": 1.358, "p95_ms": 1.358}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.934, "count": 14, "max_ms": 39.467, "p95_ms": 39.467}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.03, "count": 14, "max_ms": 47.041, "p95_ms": 47.041}, "yolo11_640_fire_11class": {"avg_ms": 47.188, "count": 14, "max_ms": 49.608, "p95_ms": 49.608}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.249, "count": 14, "max_ms": 47.974, "p95_ms": 47.974}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.565, "count": 14, "max_ms": 80.21, "p95_ms": 80.21}, "read_interval": {"avg_ms": 2266.021, "count": 14, "max_ms": 16530.515, "p95_ms": 16530.515}, "triton_wait": {"avg_ms": 47.83, "count": 14, "max_ms": 49.621, "p95_ms": 49.621}, "visualize": {"avg_ms": 3.619, "count": 14, "max_ms": 6.081, "p95_ms": 6.081}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.678, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10922}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10491.805, "limit_mb": 8796093022207.996}, "detail": {"pid": 68091, "restart_count": 140, "start_count": 141}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 140, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:05:33+09:00", "last_error_summary": "[h264 @ 0x56102bb91b40] error while decoding MB 16 13, bytestream -5 [in#0/rtsp @ 0x56102bb5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56102bb5ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:05:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:05:36+09:00", "pid": 68091, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.234, "name": "ffmpeg", "pid": 68091, "rss_mb": 156.234, "state": "R (running)", "threads": 49}, "restart_count": 140, "restart_count_per_hour": 6, "rtsp_timeout_count": 226, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.945, "sample_count": 2536, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1310657, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:05:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.754, "last_frame_pushed_at": "2026-07-01T00:05:33+09:00", "last_frame_read_age_sec": 4.836, "last_frame_read_at": "2026-07-01T00:05:33+09:00", "mjpeg": {"age_sec": 4.737, "exists": true, "mtime": "2026-07-01T00:05:33.282212+09:00", "mtime_epoch": 1782831933.2822123, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101603}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:05:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1678.203, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.625, "pushed": 0.625, "read": 0.625}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.453, "count": 14, "max_ms": 21.165, "p95_ms": 21.165}, "gstreamer_push": {"avg_ms": 1.226, "count": 14, "max_ms": 1.358, "p95_ms": 1.358}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.934, "count": 14, "max_ms": 39.467, "p95_ms": 39.467}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.03, "count": 14, "max_ms": 47.041, "p95_ms": 47.041}, "yolo11_640_fire_11class": {"avg_ms": 47.188, "count": 14, "max_ms": 49.608, "p95_ms": 49.608}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.249, "count": 14, "max_ms": 47.974, "p95_ms": 47.974}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.565, "count": 14, "max_ms": 80.21, "p95_ms": 80.21}, "read_interval": {"avg_ms": 2266.021, "count": 14, "max_ms": 16530.515, "p95_ms": 16530.515}, "triton_wait": {"avg_ms": 47.83, "count": 14, "max_ms": 49.621, "p95_ms": 49.621}, "visualize": {"avg_ms": 3.619, "count": 14, "max_ms": 6.081, "p95_ms": 6.081}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.416, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9708}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10384.758, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f3e4f6e400] cabac decode of qscale diff failed at 20 23 [h264 @ 0x55f3e4f6e400] error while decoding MB 20 23, bytestream 827 [h264 @ 0x55f3e589c680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f3e589c680] error while decoding MB 0 44, bytestream 9366 [h264 @ 0x55f3e4f73dc0] error while decoding MB 36 38, bytestream -8 [h264 @ 0x55f3e4f73dc0] cabac decode of qscale diff failed at 44 43 [h264 @ 0x55f3e4f73dc0] error while decoding MB 44 43, bytestream 5734 [h264 ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 348, "h264_decode_error_count": 1111, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:05:51+09:00", "last_error_summary": "[h264 @ 0x55f3e4f6e400] cabac decode of qscale diff failed at 20 23 [h264 @ 0x55f3e4f6e400] error while decoding MB 20 23, bytestream 827 [h264 @ 0x55f3e589c680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f3e589c680] error while decoding MB 0 44, bytestream 9366 [h264 @ 0x55f3e4f73dc0] error while decoding MB 36 38, bytestream -8 [h264 @ 0x55f3e4f73dc0] cabac decode of qscale diff failed at 44 43 [h264 @ 0x55f3e4f73dc0] error while decoding MB 44 43, bytestream 5734 [h264 ", "last_exit_at": "2026-06-30T23:53:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 760.876, "last_started_at": "2026-06-30T23:53:10+09:00", "pid": 67832, "process": {"exists": false, "pid": 67832}, "restart_count": 381, "restart_count_per_hour": 3, "rtsp_timeout_count": 314, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 382}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.825, "sample_count": 2746, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1165041, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:05:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-01T00:05:51+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-01T00:05:51+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T00:05:51.190555+09:00", "mtime_epoch": 1782831951.1905553, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120481}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:05:51+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4920.082, "name": "celery", "pid": 267, "rss_mb": 4920.082, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.009, "pushed": 4.009, "read": 4.009}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.562, "count": 120, "max_ms": 25.403, "p95_ms": 22.997}, "gstreamer_push": {"avg_ms": 1.218, "count": 120, "max_ms": 1.343, "p95_ms": 1.31}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.154, "count": 120, "max_ms": 33.565, "p95_ms": 31.382}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.062, "count": 120, "max_ms": 49.68, "p95_ms": 45.751}, "yolo11_640_fire_11class": {"avg_ms": 43.451, "count": 120, "max_ms": 49.128, "p95_ms": 44.89}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.908, "count": 120, "max_ms": 50.687, "p95_ms": 46.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.429, "count": 120, "max_ms": 84.165, "p95_ms": 76.33}, "read_interval": {"avg_ms": 249.47, "count": 120, "max_ms": 10126.775, "p95_ms": 92.19}, "triton_wait": {"avg_ms": 44.957, "count": 120, "max_ms": 50.699, "p95_ms": 46.951}, "visualize": {"avg_ms": 3.374, "count": 120, "max_ms": 3.685, "p95_ms": 3.554}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9708}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10384.566, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:05:51+09:00", "last_error_summary": "[h264 @ 0x55f3e4f6e400] cabac decode of qscale diff failed at 20 23 [h264 @ 0x55f3e4f6e400] error while decoding MB 20 23, bytestream 827 [h264 @ 0x55f3e589c680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f3e589c680] error while decoding MB 0 44, bytestream 9366 [h264 @ 0x55f3e4f73dc0] error while decoding MB 36 38, bytestream -8 [h264 @ 0x55f3e4f73dc0] cabac decode of qscale diff failed at 44 43 [h264 @ 0x55f3e4f73dc0] error while decoding MB 44 43, bytestream 5734 [h264 ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 348, "h264_decode_error_count": 1111, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:05:51+09:00", "last_error_summary": "[h264 @ 0x55f3e4f6e400] cabac decode of qscale diff failed at 20 23 [h264 @ 0x55f3e4f6e400] error while decoding MB 20 23, bytestream 827 [h264 @ 0x55f3e589c680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f3e589c680] error while decoding MB 0 44, bytestream 9366 [h264 @ 0x55f3e4f73dc0] error while decoding MB 36 38, bytestream -8 [h264 @ 0x55f3e4f73dc0] cabac decode of qscale diff failed at 44 43 [h264 @ 0x55f3e4f73dc0] error while decoding MB 44 43, bytestream 5734 [h264 ", "last_exit_at": "2026-07-01T00:05:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 760.906, "last_started_at": "2026-06-30T23:53:10+09:00", "pid": 67832, "process": {"exists": false, "pid": 67832}, "restart_count": 381, "restart_count_per_hour": 3, "rtsp_timeout_count": 314, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 382}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.825, "sample_count": 2747, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1165042, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:05:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T00:05:51+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-01T00:05:51+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-01T00:05:51.190555+09:00", "mtime_epoch": 1782831951.1905553, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120481}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:05:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4920.082, "name": "celery", "pid": 267, "rss_mb": 4920.082, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.039, "pushed": 4.039, "read": 4.005}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.609, "count": 121, "max_ms": 26.253, "p95_ms": 23.047}, "gstreamer_push": {"avg_ms": 1.219, "count": 121, "max_ms": 1.364, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.155, "count": 121, "max_ms": 33.565, "p95_ms": 31.382}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.076, "count": 121, "max_ms": 49.68, "p95_ms": 45.751}, "yolo11_640_fire_11class": {"avg_ms": 43.461, "count": 121, "max_ms": 49.128, "p95_ms": 44.89}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.919, "count": 121, "max_ms": 50.687, "p95_ms": 46.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.512, "count": 121, "max_ms": 84.165, "p95_ms": 76.427}, "read_interval": {"avg_ms": 249.47, "count": 120, "max_ms": 10126.775, "p95_ms": 92.19}, "triton_wait": {"avg_ms": 44.967, "count": 121, "max_ms": 50.699, "p95_ms": 46.951}, "visualize": {"avg_ms": 3.395, "count": 121, "max_ms": 5.884, "p95_ms": 3.571}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9708}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10384.352, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 348, "h264_decode_error_count": 1111, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:05:51+09:00", "last_error_summary": "[h264 @ 0x55f3e4f6e400] cabac decode of qscale diff failed at 20 23 [h264 @ 0x55f3e4f6e400] error while decoding MB 20 23, bytestream 827 [h264 @ 0x55f3e589c680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f3e589c680] error while decoding MB 0 44, bytestream 9366 [h264 @ 0x55f3e4f73dc0] error while decoding MB 36 38, bytestream -8 [h264 @ 0x55f3e4f73dc0] cabac decode of qscale diff failed at 44 43 [h264 @ 0x55f3e4f73dc0] error while decoding MB 44 43, bytestream 5734 [h264 ", "last_exit_at": "2026-07-01T00:05:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 760.914, "last_started_at": "2026-06-30T23:53:10+09:00", "pid": 67832, "process": {"exists": false, "pid": 67832}, "restart_count": 381, "restart_count_per_hour": 3, "rtsp_timeout_count": 314, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 382}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.825, "sample_count": 2746, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1165042, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:05:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-01T00:05:51+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T00:05:51+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-01T00:05:51.190555+09:00", "mtime_epoch": 1782831951.1905553, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120481}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:05:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4920.082, "name": "celery", "pid": 267, "rss_mb": 4920.082, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.038, "pushed": 4.038, "read": 4.004}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.609, "count": 121, "max_ms": 26.253, "p95_ms": 23.047}, "gstreamer_push": {"avg_ms": 1.219, "count": 121, "max_ms": 1.364, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.155, "count": 121, "max_ms": 33.565, "p95_ms": 31.382}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.076, "count": 121, "max_ms": 49.68, "p95_ms": 45.751}, "yolo11_640_fire_11class": {"avg_ms": 43.461, "count": 121, "max_ms": 49.128, "p95_ms": 44.89}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.919, "count": 121, "max_ms": 50.687, "p95_ms": 46.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.512, "count": 121, "max_ms": 84.165, "p95_ms": 76.427}, "read_interval": {"avg_ms": 249.47, "count": 120, "max_ms": 10126.775, "p95_ms": 92.19}, "triton_wait": {"avg_ms": 44.967, "count": 121, "max_ms": 50.699, "p95_ms": 46.951}, "visualize": {"avg_ms": 3.395, "count": 121, "max_ms": 5.884, "p95_ms": 3.571}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9708}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10492.168, "limit_mb": 8796093022207.996}, "detail": {"pid": 68243, "restart_count": 382, "start_count": 383}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 348, "h264_decode_error_count": 1111, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:05:51+09:00", "last_error_summary": "[h264 @ 0x55f3e4f6e400] cabac decode of qscale diff failed at 20 23 [h264 @ 0x55f3e4f6e400] error while decoding MB 20 23, bytestream 827 [h264 @ 0x55f3e589c680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f3e589c680] error while decoding MB 0 44, bytestream 9366 [h264 @ 0x55f3e4f73dc0] error while decoding MB 36 38, bytestream -8 [h264 @ 0x55f3e4f73dc0] cabac decode of qscale diff failed at 44 43 [h264 @ 0x55f3e4f73dc0] error while decoding MB 44 43, bytestream 5734 [h264 ", "last_exit_at": "2026-07-01T00:05:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:05:53+09:00", "pid": 68243, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.727, "name": "ffmpeg", "pid": 68243, "rss_mb": 136.727, "state": "R (running)", "threads": 106}, "restart_count": 382, "restart_count_per_hour": 4, "rtsp_timeout_count": 314, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 383}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.803, "sample_count": 2707, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1165042, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:05:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.488, "last_frame_pushed_at": "2026-07-01T00:05:51+09:00", "last_frame_read_age_sec": 4.574, "last_frame_read_at": "2026-07-01T00:05:51+09:00", "mjpeg": {"age_sec": 4.472, "exists": true, "mtime": "2026-07-01T00:05:51.286557+09:00", "mtime_epoch": 1782831951.2865572, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120340}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:05:55+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4920.082, "name": "celery", "pid": 267, "rss_mb": 4920.082, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 70, "pushed_frames": 70, "read_frames": 69, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.339, "pushed": 2.339, "read": 2.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.908, "count": 70, "max_ms": 26.253, "p95_ms": 23.699}, "gstreamer_push": {"avg_ms": 1.233, "count": 70, "max_ms": 1.364, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.25, "count": 70, "max_ms": 33.565, "p95_ms": 31.4}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.044, "count": 70, "max_ms": 49.68, "p95_ms": 45.746}, "yolo11_640_fire_11class": {"avg_ms": 43.483, "count": 70, "max_ms": 49.128, "p95_ms": 45.074}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.921, "count": 70, "max_ms": 50.687, "p95_ms": 46.429}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.865, "count": 70, "max_ms": 84.165, "p95_ms": 77.381}, "read_interval": {"avg_ms": 368.553, "count": 69, "max_ms": 10126.775, "p95_ms": 94.874}, "triton_wait": {"avg_ms": 44.952, "count": 70, "max_ms": 50.699, "p95_ms": 46.441}, "visualize": {"avg_ms": 3.417, "count": 70, "max_ms": 5.884, "p95_ms": 3.571}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10940}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10326.809, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56344b8b8ac0] error while decoding MB 72 29, bytestream -15 [in#0/rtsp @ 0x56344b88ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56344b88ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:10:08+09:00", "last_error_summary": "[h264 @ 0x56344b8b8ac0] error while decoding MB 72 29, bytestream -15 [in#0/rtsp @ 0x56344b88ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56344b88ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:05:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 272.136, "last_started_at": "2026-07-01T00:05:36+09:00", "pid": 68091, "process": {"exists": false, "pid": 68091}, "restart_count": 140, "restart_count_per_hour": 6, "rtsp_timeout_count": 228, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.034, "sample_count": 2272, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1312915, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:10:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T00:10:08+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-01T00:10:08+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T00:10:08.093018+09:00", "mtime_epoch": 1782832208.0930183, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103485}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:10:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1679.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.699, "pushed": 0.699, "read": 0.757}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.11, "count": 12, "max_ms": 21.375, "p95_ms": 21.375}, "gstreamer_push": {"avg_ms": 1.214, "count": 12, "max_ms": 1.341, "p95_ms": 1.341}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.02, "count": 12, "max_ms": 51.507, "p95_ms": 51.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.049, "count": 12, "max_ms": 55.451, "p95_ms": 55.451}, "yolo11_640_fire_11class": {"avg_ms": 47.797, "count": 12, "max_ms": 52.449, "p95_ms": 52.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.014, "count": 12, "max_ms": 53.944, "p95_ms": 53.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.327, "count": 12, "max_ms": 82.567, "p95_ms": 82.567}, "read_interval": {"avg_ms": 2520.36, "count": 13, "max_ms": 16109.856, "p95_ms": 16109.856}, "triton_wait": {"avg_ms": 49.152, "count": 12, "max_ms": 55.461, "p95_ms": 55.461}, "visualize": {"avg_ms": 3.29, "count": 12, "max_ms": 3.514, "p95_ms": 3.514}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.169, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10940}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10326.582, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:10:08+09:00", "last_error_summary": "[h264 @ 0x56344b8b8ac0] error while decoding MB 72 29, bytestream -15 [in#0/rtsp @ 0x56344b88ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56344b88ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:10:08+09:00", "last_error_summary": "[h264 @ 0x56344b8b8ac0] error while decoding MB 72 29, bytestream -15 [in#0/rtsp @ 0x56344b88ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56344b88ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:10:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 272.159, "last_started_at": "2026-07-01T00:05:36+09:00", "pid": 68091, "process": {"exists": false, "pid": 68091}, "restart_count": 140, "restart_count_per_hour": 6, "rtsp_timeout_count": 228, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.034, "sample_count": 2273, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1312916, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:10:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T00:10:08+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-01T00:10:08+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-01T00:10:08.093018+09:00", "mtime_epoch": 1782832208.0930183, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103485}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:10:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1679.434, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.756, "pushed": 0.756, "read": 0.756}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.345, "count": 13, "max_ms": 23.165, "p95_ms": 23.165}, "gstreamer_push": {"avg_ms": 1.226, "count": 13, "max_ms": 1.376, "p95_ms": 1.376}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.945, "count": 13, "max_ms": 51.507, "p95_ms": 51.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.026, "count": 13, "max_ms": 55.451, "p95_ms": 55.451}, "yolo11_640_fire_11class": {"avg_ms": 47.793, "count": 13, "max_ms": 52.449, "p95_ms": 52.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.962, "count": 13, "max_ms": 53.944, "p95_ms": 53.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.727, "count": 13, "max_ms": 82.567, "p95_ms": 82.567}, "read_interval": {"avg_ms": 2520.36, "count": 13, "max_ms": 16109.856, "p95_ms": 16109.856}, "triton_wait": {"avg_ms": 49.091, "count": 13, "max_ms": 55.461, "p95_ms": 55.461}, "visualize": {"avg_ms": 3.514, "count": 13, "max_ms": 6.194, "p95_ms": 6.194}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.192, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10940}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10326.695, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:10:08+09:00", "last_error_summary": "[h264 @ 0x56344b8b8ac0] error while decoding MB 72 29, bytestream -15 [in#0/rtsp @ 0x56344b88ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56344b88ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:10:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 272.166, "last_started_at": "2026-07-01T00:05:36+09:00", "pid": 68091, "process": {"exists": false, "pid": 68091}, "restart_count": 140, "restart_count_per_hour": 6, "rtsp_timeout_count": 228, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.034, "sample_count": 2273, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1312916, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:10:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T00:10:08+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-01T00:10:08+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-01T00:10:08.093018+09:00", "mtime_epoch": 1782832208.0930183, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103485}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:10:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1679.434, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.756, "pushed": 0.756, "read": 0.756}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.345, "count": 13, "max_ms": 23.165, "p95_ms": 23.165}, "gstreamer_push": {"avg_ms": 1.226, "count": 13, "max_ms": 1.376, "p95_ms": 1.376}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.945, "count": 13, "max_ms": 51.507, "p95_ms": 51.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.026, "count": 13, "max_ms": 55.451, "p95_ms": 55.451}, "yolo11_640_fire_11class": {"avg_ms": 47.793, "count": 13, "max_ms": 52.449, "p95_ms": 52.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.962, "count": 13, "max_ms": 53.944, "p95_ms": 53.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.727, "count": 13, "max_ms": 82.567, "p95_ms": 82.567}, "read_interval": {"avg_ms": 2520.36, "count": 13, "max_ms": 16109.856, "p95_ms": 16109.856}, "triton_wait": {"avg_ms": 49.091, "count": 13, "max_ms": 55.461, "p95_ms": 55.461}, "visualize": {"avg_ms": 3.514, "count": 13, "max_ms": 6.194, "p95_ms": 6.194}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.2, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10940}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10338.73, "limit_mb": 8796093022207.996}, "detail": {"pid": 68350, "restart_count": 141, "start_count": 142}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:10:08+09:00", "last_error_summary": "[h264 @ 0x56344b8b8ac0] error while decoding MB 72 29, bytestream -15 [in#0/rtsp @ 0x56344b88ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56344b88ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:10:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:10:10+09:00", "pid": 68350, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.855, "name": "ffmpeg", "pid": 68350, "rss_mb": 49.445, "state": "R (running)", "threads": 1}, "restart_count": 141, "restart_count_per_hour": 7, "rtsp_timeout_count": 228, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.034, "sample_count": 2273, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1312916, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:10:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.624, "last_frame_pushed_at": "2026-07-01T00:10:08+09:00", "last_frame_read_age_sec": 4.709, "last_frame_read_at": "2026-07-01T00:10:08+09:00", "mjpeg": {"age_sec": 4.611, "exists": true, "mtime": "2026-07-01T00:10:08.185021+09:00", "mtime_epoch": 1782832208.185021, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103862}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:10:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1679.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.596, "pushed": 0.596, "read": 0.596}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.345, "count": 13, "max_ms": 23.165, "p95_ms": 23.165}, "gstreamer_push": {"avg_ms": 1.226, "count": 13, "max_ms": 1.376, "p95_ms": 1.376}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.945, "count": 13, "max_ms": 51.507, "p95_ms": 51.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.026, "count": 13, "max_ms": 55.451, "p95_ms": 55.451}, "yolo11_640_fire_11class": {"avg_ms": 47.793, "count": 13, "max_ms": 52.449, "p95_ms": 52.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.962, "count": 13, "max_ms": 53.944, "p95_ms": 53.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.727, "count": 13, "max_ms": 82.567, "p95_ms": 82.567}, "read_interval": {"avg_ms": 2520.36, "count": 13, "max_ms": 16109.856, "p95_ms": 16109.856}, "triton_wait": {"avg_ms": 49.091, "count": 13, "max_ms": 55.461, "p95_ms": 55.461}, "visualize": {"avg_ms": 3.514, "count": 13, "max_ms": 6.194, "p95_ms": 6.194}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.811, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10958}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10644.184, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:10:08+09:00", "last_error_summary": "[h264 @ 0x56344b8b8ac0] error while decoding MB 72 29, bytestream -15 [in#0/rtsp @ 0x56344b88ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56344b88ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:10:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 247.036, "last_started_at": "2026-07-01T00:10:10+09:00", "pid": 68350, "process": {"exists": true, "fd_count": 5, "hwm_mb": 330.168, "name": "ffmpeg", "pid": 68350, "rss_mb": 330.168, "state": "S (sleeping)", "threads": 106}, "restart_count": 141, "restart_count_per_hour": 6, "rtsp_timeout_count": 228, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.236, "sample_count": 2228, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1314965, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:13:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.055, "last_frame_pushed_at": "2026-07-01T00:13:47+09:00", "last_frame_read_age_sec": 30.162, "last_frame_read_at": "2026-07-01T00:13:47+09:00", "mjpeg": {"age_sec": 30.04, "exists": true, "mtime": "2026-07-01T00:13:47.790807+09:00", "mtime_epoch": 1782832427.7908068, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103777}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:14:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1680.418, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10958}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10643.66, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:10:08+09:00", "last_error_summary": "[h264 @ 0x56344b8b8ac0] error while decoding MB 72 29, bytestream -15 [in#0/rtsp @ 0x56344b88ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56344b88ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:10:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 247.048, "last_started_at": "2026-07-01T00:10:10+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 141, "restart_count_per_hour": 6, "rtsp_timeout_count": 228, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.236, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1314965, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:13:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.068, "last_frame_pushed_at": "2026-07-01T00:13:47+09:00", "last_frame_read_age_sec": 30.174, "last_frame_read_at": "2026-07-01T00:13:47+09:00", "mjpeg": {"age_sec": 30.052, "exists": true, "mtime": "2026-07-01T00:13:47.790807+09:00", "mtime_epoch": 1782832427.7908068, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103777}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:14:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1680.418, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10958}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10343.176, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d1fbfc2f00] error while decoding MB 95 31, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 142, "h264_decode_error_count": 147, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:14:17+09:00", "last_error_summary": "[h264 @ 0x55d1fbfc2f00] error while decoding MB 95 31, bytestream -9", "last_exit_at": "2026-07-01T00:10:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 247.099, "last_started_at": "2026-07-01T00:10:10+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 141, "restart_count_per_hour": 6, "rtsp_timeout_count": 228, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.236, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1314965, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:13:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.119, "last_frame_pushed_at": "2026-07-01T00:13:47+09:00", "last_frame_read_age_sec": 30.225, "last_frame_read_at": "2026-07-01T00:13:47+09:00", "mjpeg": {"age_sec": 30.103, "exists": true, "mtime": "2026-07-01T00:13:47.790807+09:00", "mtime_epoch": 1782832427.7908068, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103777}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:14:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1680.418, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10958}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10364.008, "limit_mb": 8796093022207.996}, "detail": {"pid": 68502, "restart_count": 142, "start_count": 143}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 142, "h264_decode_error_count": 147, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:14:17+09:00", "last_error_summary": "[h264 @ 0x55d1fbfc2f00] error while decoding MB 95 31, bytestream -9", "last_exit_at": "2026-07-01T00:10:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:14:20+09:00", "pid": 68502, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.109, "name": "ffmpeg", "pid": 68502, "rss_mb": 57.109, "state": "S (sleeping)", "threads": 1}, "restart_count": 142, "restart_count_per_hour": 7, "rtsp_timeout_count": 228, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.245, "sample_count": 2174, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1314965, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:13:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 34.947, "last_frame_pushed_at": "2026-07-01T00:13:47+09:00", "last_frame_read_age_sec": 35.054, "last_frame_read_at": "2026-07-01T00:13:47+09:00", "mjpeg": {"age_sec": 34.932, "exists": true, "mtime": "2026-07-01T00:13:47.790807+09:00", "mtime_epoch": 1782832427.7908068, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103777}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:14:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1680.418, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9775}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10391.438, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5630aa10de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5630aa10de80] error while decoding MB 0 34, bytestream 2015 [h264 @ 0x5630aa0470c0] error while decoding MB 1 14, bytestream -14 [h264 @ 0x5630aa97e980] top block unavailable for requested intra mode [h264 @ 0x5630aa97e980] error while decoding MB 22 0, bytestream 8757 [h264 @ 0x5630aa10de80] error while decoding MB 73 2, bytestream -13 [h264 @ 0x5630aab1c500] error while decoding MB 3 27, bytestream -18 [h26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 349, "h264_decode_error_count": 1117, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:20:10+09:00", "last_error_summary": "[h264 @ 0x5630aa10de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5630aa10de80] error while decoding MB 0 34, bytestream 2015 [h264 @ 0x5630aa0470c0] error while decoding MB 1 14, bytestream -14 [h264 @ 0x5630aa97e980] top block unavailable for requested intra mode [h264 @ 0x5630aa97e980] error while decoding MB 22 0, bytestream 8757 [h264 @ 0x5630aa10de80] error while decoding MB 73 2, bytestream -13 [h264 @ 0x5630aab1c500] error while decoding MB 3 27, bytestream -18 [h26", "last_exit_at": "2026-07-01T00:05:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 856.88, "last_started_at": "2026-07-01T00:05:53+09:00", "pid": 68243, "process": {"exists": false, "pid": 68243}, "restart_count": 382, "restart_count_per_hour": 3, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 383}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.214, "sample_count": 2688, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1173104, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:20:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T00:20:10+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-01T00:20:10+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-01T00:20:10.579947+09:00", "mtime_epoch": 1782832810.5799465, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119905}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:20:10+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4923.691, "name": "celery", "pid": 267, "rss_mb": 4923.691, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 118, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.934, "pushed": 3.934, "read": 3.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.476, "count": 118, "max_ms": 28.973, "p95_ms": 24.442}, "gstreamer_push": {"avg_ms": 1.276, "count": 118, "max_ms": 1.63, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.651, "count": 118, "max_ms": 36.026, "p95_ms": 33.813}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 45.222, "count": 118, "max_ms": 49.642, "p95_ms": 48.512}, "yolo11_640_fire_11class": {"avg_ms": 44.444, "count": 118, "max_ms": 49.825, "p95_ms": 46.966}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.705, "count": 118, "max_ms": 50.726, "p95_ms": 48.427}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.506, "count": 118, "max_ms": 82.273, "p95_ms": 80.212}, "read_interval": {"avg_ms": 254.298, "count": 118, "max_ms": 10126.22, "p95_ms": 96.842}, "triton_wait": {"avg_ms": 46.051, "count": 118, "max_ms": 50.738, "p95_ms": 48.74}, "visualize": {"avg_ms": 3.384, "count": 118, "max_ms": 4.417, "p95_ms": 3.583}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9775}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10391.297, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:20:10+09:00", "last_error_summary": "[h264 @ 0x5630aa10de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5630aa10de80] error while decoding MB 0 34, bytestream 2015 [h264 @ 0x5630aa0470c0] error while decoding MB 1 14, bytestream -14 [h264 @ 0x5630aa97e980] top block unavailable for requested intra mode [h264 @ 0x5630aa97e980] error while decoding MB 22 0, bytestream 8757 [h264 @ 0x5630aa10de80] error while decoding MB 73 2, bytestream -13 [h264 @ 0x5630aab1c500] error while decoding MB 3 27, bytestream -18 [h26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 349, "h264_decode_error_count": 1117, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:20:10+09:00", "last_error_summary": "[h264 @ 0x5630aa10de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5630aa10de80] error while decoding MB 0 34, bytestream 2015 [h264 @ 0x5630aa0470c0] error while decoding MB 1 14, bytestream -14 [h264 @ 0x5630aa97e980] top block unavailable for requested intra mode [h264 @ 0x5630aa97e980] error while decoding MB 22 0, bytestream 8757 [h264 @ 0x5630aa10de80] error while decoding MB 73 2, bytestream -13 [h264 @ 0x5630aab1c500] error while decoding MB 3 27, bytestream -18 [h26", "last_exit_at": "2026-07-01T00:20:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 856.905, "last_started_at": "2026-07-01T00:05:53+09:00", "pid": 68243, "process": {"exists": false, "pid": 68243}, "restart_count": 382, "restart_count_per_hour": 3, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 383}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.214, "sample_count": 2688, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1173105, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:20:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T00:20:10+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-01T00:20:10+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-01T00:20:10.579947+09:00", "mtime_epoch": 1782832810.5799465, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119905}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:20:10+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4923.691, "name": "celery", "pid": 267, "rss_mb": 4923.691, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.943, "pushed": 3.943, "read": 3.91}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.481, "count": 118, "max_ms": 28.973, "p95_ms": 24.442}, "gstreamer_push": {"avg_ms": 1.276, "count": 118, "max_ms": 1.63, "p95_ms": 1.405}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.646, "count": 118, "max_ms": 36.026, "p95_ms": 33.813}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 45.193, "count": 118, "max_ms": 49.642, "p95_ms": 48.512}, "yolo11_640_fire_11class": {"avg_ms": 44.437, "count": 118, "max_ms": 49.825, "p95_ms": 46.966}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.7, "count": 118, "max_ms": 50.726, "p95_ms": 48.427}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.515, "count": 118, "max_ms": 82.273, "p95_ms": 80.212}, "read_interval": {"avg_ms": 255.674, "count": 117, "max_ms": 10126.22, "p95_ms": 96.842}, "triton_wait": {"avg_ms": 46.027, "count": 118, "max_ms": 50.738, "p95_ms": 48.74}, "visualize": {"avg_ms": 3.409, "count": 118, "max_ms": 6.151, "p95_ms": 3.651}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9775}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10391.27, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 349, "h264_decode_error_count": 1117, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:20:10+09:00", "last_error_summary": "[h264 @ 0x5630aa10de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5630aa10de80] error while decoding MB 0 34, bytestream 2015 [h264 @ 0x5630aa0470c0] error while decoding MB 1 14, bytestream -14 [h264 @ 0x5630aa97e980] top block unavailable for requested intra mode [h264 @ 0x5630aa97e980] error while decoding MB 22 0, bytestream 8757 [h264 @ 0x5630aa10de80] error while decoding MB 73 2, bytestream -13 [h264 @ 0x5630aab1c500] error while decoding MB 3 27, bytestream -18 [h26", "last_exit_at": "2026-07-01T00:20:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 856.913, "last_started_at": "2026-07-01T00:05:53+09:00", "pid": 68243, "process": {"exists": false, "pid": 68243}, "restart_count": 382, "restart_count_per_hour": 3, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 383}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.214, "sample_count": 2688, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1173105, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:20:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T00:20:10+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-01T00:20:10+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T00:20:10.579947+09:00", "mtime_epoch": 1782832810.5799465, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119905}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:20:10+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4923.691, "name": "celery", "pid": 267, "rss_mb": 4923.691, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.942, "pushed": 3.942, "read": 3.909}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.481, "count": 118, "max_ms": 28.973, "p95_ms": 24.442}, "gstreamer_push": {"avg_ms": 1.276, "count": 118, "max_ms": 1.63, "p95_ms": 1.405}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.646, "count": 118, "max_ms": 36.026, "p95_ms": 33.813}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 45.193, "count": 118, "max_ms": 49.642, "p95_ms": 48.512}, "yolo11_640_fire_11class": {"avg_ms": 44.437, "count": 118, "max_ms": 49.825, "p95_ms": 46.966}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.7, "count": 118, "max_ms": 50.726, "p95_ms": 48.427}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.515, "count": 118, "max_ms": 82.273, "p95_ms": 80.212}, "read_interval": {"avg_ms": 255.674, "count": 117, "max_ms": 10126.22, "p95_ms": 96.842}, "triton_wait": {"avg_ms": 46.027, "count": 118, "max_ms": 50.738, "p95_ms": 48.74}, "visualize": {"avg_ms": 3.409, "count": 118, "max_ms": 6.151, "p95_ms": 3.651}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9775}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10507.148, "limit_mb": 8796093022207.996}, "detail": {"pid": 68654, "restart_count": 383, "start_count": 384}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 349, "h264_decode_error_count": 1117, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:20:10+09:00", "last_error_summary": "[h264 @ 0x5630aa10de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5630aa10de80] error while decoding MB 0 34, bytestream 2015 [h264 @ 0x5630aa0470c0] error while decoding MB 1 14, bytestream -14 [h264 @ 0x5630aa97e980] top block unavailable for requested intra mode [h264 @ 0x5630aa97e980] error while decoding MB 22 0, bytestream 8757 [h264 @ 0x5630aa10de80] error while decoding MB 73 2, bytestream -13 [h264 @ 0x5630aab1c500] error while decoding MB 3 27, bytestream -18 [h26", "last_exit_at": "2026-07-01T00:20:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:20:13+09:00", "pid": 68654, "process": {"exists": true, "fd_count": 5, "hwm_mb": 146.785, "name": "ffmpeg", "pid": 68654, "rss_mb": 146.785, "state": "S (sleeping)", "threads": 106}, "restart_count": 383, "restart_count_per_hour": 4, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 384}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.238, "sample_count": 2640, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1173105, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:20:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.392, "last_frame_pushed_at": "2026-07-01T00:20:10+09:00", "last_frame_read_age_sec": 4.474, "last_frame_read_at": "2026-07-01T00:20:10+09:00", "mjpeg": {"age_sec": 4.377, "exists": true, "mtime": "2026-07-01T00:20:10.671949+09:00", "mtime_epoch": 1782832810.6719487, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119926}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:20:15+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4923.691, "name": "celery", "pid": 267, "rss_mb": 4923.691, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 70, "pushed_frames": 70, "read_frames": 70, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.334, "pushed": 2.334, "read": 2.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.56, "count": 70, "max_ms": 28.973, "p95_ms": 25.31}, "gstreamer_push": {"avg_ms": 1.281, "count": 70, "max_ms": 1.63, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.773, "count": 70, "max_ms": 36.026, "p95_ms": 33.813}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 45.312, "count": 70, "max_ms": 49.154, "p95_ms": 48.049}, "yolo11_640_fire_11class": {"avg_ms": 44.559, "count": 70, "max_ms": 48.672, "p95_ms": 46.966}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.536, "count": 70, "max_ms": 48.001, "p95_ms": 46.859}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.592, "count": 70, "max_ms": 82.273, "p95_ms": 80.212}, "read_interval": {"avg_ms": 365.797, "count": 70, "max_ms": 10126.22, "p95_ms": 96.659}, "triton_wait": {"avg_ms": 46.002, "count": 70, "max_ms": 49.167, "p95_ms": 48.061}, "visualize": {"avg_ms": 3.427, "count": 70, "max_ms": 6.151, "p95_ms": 3.651}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10986}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10334.258, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55fde0d53240] error while decoding MB 117 36, bytestream -17 [in#0/rtsp @ 0x55fde03b4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fde03b4e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:20:41+09:00", "last_error_summary": "[h264 @ 0x55fde0d53240] error while decoding MB 117 36, bytestream -17 [in#0/rtsp @ 0x55fde03b4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fde03b4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:10:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 381.15, "last_started_at": "2026-07-01T00:14:20+09:00", "pid": 68502, "process": {"exists": false, "pid": 68502}, "restart_count": 142, "restart_count_per_hour": 7, "rtsp_timeout_count": 230, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.384, "sample_count": 2592, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1318325, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:20:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-01T00:20:41+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-01T00:20:41+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-01T00:20:41.816658+09:00", "mtime_epoch": 1782832841.8166578, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90732}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:20:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1682.145, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.747, "pushed": 0.747, "read": 0.804}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.708, "count": 13, "max_ms": 19.998, "p95_ms": 19.998}, "gstreamer_push": {"avg_ms": 1.255, "count": 13, "max_ms": 1.289, "p95_ms": 1.289}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.801, "count": 13, "max_ms": 54.983, "p95_ms": 54.983}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.131, "count": 13, "max_ms": 68.028, "p95_ms": 68.028}, "yolo11_640_fire_11class": {"avg_ms": 59.236, "count": 13, "max_ms": 64.225, "p95_ms": 64.225}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.098, "count": 13, "max_ms": 73.845, "p95_ms": 73.845}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.068, "count": 13, "max_ms": 100.693, "p95_ms": 100.693}, "read_interval": {"avg_ms": 2349.346, "count": 14, "max_ms": 16052.148, "p95_ms": 16052.148}, "triton_wait": {"avg_ms": 67.111, "count": 13, "max_ms": 73.856, "p95_ms": 73.856}, "visualize": {"avg_ms": 3.533, "count": 13, "max_ms": 3.864, "p95_ms": 3.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.402, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10986}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10333.414, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:20:41+09:00", "last_error_summary": "[h264 @ 0x55fde0d53240] error while decoding MB 117 36, bytestream -17 [in#0/rtsp @ 0x55fde03b4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fde03b4e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:20:41+09:00", "last_error_summary": "[h264 @ 0x55fde0d53240] error while decoding MB 117 36, bytestream -17 [in#0/rtsp @ 0x55fde03b4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fde03b4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:20:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 381.199, "last_started_at": "2026-07-01T00:14:20+09:00", "pid": 68502, "process": {"exists": false, "pid": 68502}, "restart_count": 142, "restart_count_per_hour": 7, "rtsp_timeout_count": 230, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.384, "sample_count": 2593, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1318326, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:20:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T00:20:41+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-01T00:20:41+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-01T00:20:41.816658+09:00", "mtime_epoch": 1782832841.8166578, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90732}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:20:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1682.145, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.802, "pushed": 0.802, "read": 0.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.772, "count": 14, "max_ms": 19.998, "p95_ms": 19.998}, "gstreamer_push": {"avg_ms": 1.261, "count": 14, "max_ms": 1.334, "p95_ms": 1.334}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.967, "count": 14, "max_ms": 54.983, "p95_ms": 54.983}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.528, "count": 14, "max_ms": 68.028, "p95_ms": 68.028}, "yolo11_640_fire_11class": {"avg_ms": 59.595, "count": 14, "max_ms": 64.274, "p95_ms": 64.274}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.598, "count": 14, "max_ms": 74.094, "p95_ms": 74.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.637, "count": 14, "max_ms": 102.041, "p95_ms": 102.041}, "read_interval": {"avg_ms": 2349.346, "count": 14, "max_ms": 16052.148, "p95_ms": 16052.148}, "triton_wait": {"avg_ms": 67.611, "count": 14, "max_ms": 74.107, "p95_ms": 74.107}, "visualize": {"avg_ms": 3.53, "count": 14, "max_ms": 3.864, "p95_ms": 3.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.451, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10986}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10333.492, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:20:41+09:00", "last_error_summary": "[h264 @ 0x55fde0d53240] error while decoding MB 117 36, bytestream -17 [in#0/rtsp @ 0x55fde03b4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fde03b4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:20:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 381.205, "last_started_at": "2026-07-01T00:14:20+09:00", "pid": 68502, "process": {"exists": false, "pid": 68502}, "restart_count": 142, "restart_count_per_hour": 7, "rtsp_timeout_count": 230, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.384, "sample_count": 2593, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1318326, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:20:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T00:20:41+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-01T00:20:41+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-01T00:20:41.816658+09:00", "mtime_epoch": 1782832841.8166578, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90732}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:20:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1682.145, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.802, "pushed": 0.802, "read": 0.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.772, "count": 14, "max_ms": 19.998, "p95_ms": 19.998}, "gstreamer_push": {"avg_ms": 1.261, "count": 14, "max_ms": 1.334, "p95_ms": 1.334}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.967, "count": 14, "max_ms": 54.983, "p95_ms": 54.983}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.528, "count": 14, "max_ms": 68.028, "p95_ms": 68.028}, "yolo11_640_fire_11class": {"avg_ms": 59.595, "count": 14, "max_ms": 64.274, "p95_ms": 64.274}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.598, "count": 14, "max_ms": 74.094, "p95_ms": 74.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.637, "count": 14, "max_ms": 102.041, "p95_ms": 102.041}, "read_interval": {"avg_ms": 2349.346, "count": 14, "max_ms": 16052.148, "p95_ms": 16052.148}, "triton_wait": {"avg_ms": 67.611, "count": 14, "max_ms": 74.107, "p95_ms": 74.107}, "visualize": {"avg_ms": 3.53, "count": 14, "max_ms": 3.864, "p95_ms": 3.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.457, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 10986}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10354.516, "limit_mb": 8796093022207.996}, "detail": {"pid": 68761, "restart_count": 143, "start_count": 144}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:20:41+09:00", "last_error_summary": "[h264 @ 0x55fde0d53240] error while decoding MB 117 36, bytestream -17 [in#0/rtsp @ 0x55fde03b4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fde03b4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:20:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:20:44+09:00", "pid": 68761, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.723, "name": "ffmpeg", "pid": 68761, "rss_mb": 57.723, "state": "S (sleeping)", "threads": 1}, "restart_count": 143, "restart_count_per_hour": 8, "rtsp_timeout_count": 230, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 144}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.394, "sample_count": 2542, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1318326, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:20:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.747, "last_frame_pushed_at": "2026-07-01T00:20:41+09:00", "last_frame_read_age_sec": 4.851, "last_frame_read_at": "2026-07-01T00:20:41+09:00", "mjpeg": {"age_sec": 4.734, "exists": true, "mtime": "2026-07-01T00:20:41.928660+09:00", "mtime_epoch": 1782832841.9286604, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92537}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:20:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1682.145, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.631, "pushed": 0.631, "read": 0.631}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.772, "count": 14, "max_ms": 19.998, "p95_ms": 19.998}, "gstreamer_push": {"avg_ms": 1.261, "count": 14, "max_ms": 1.334, "p95_ms": 1.334}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.967, "count": 14, "max_ms": 54.983, "p95_ms": 54.983}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.528, "count": 14, "max_ms": 68.028, "p95_ms": 68.028}, "yolo11_640_fire_11class": {"avg_ms": 59.595, "count": 14, "max_ms": 64.274, "p95_ms": 64.274}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.598, "count": 14, "max_ms": 74.094, "p95_ms": 74.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.637, "count": 14, "max_ms": 102.041, "p95_ms": 102.041}, "read_interval": {"avg_ms": 2349.346, "count": 14, "max_ms": 16052.148, "p95_ms": 16052.148}, "triton_wait": {"avg_ms": 67.611, "count": 14, "max_ms": 74.107, "p95_ms": 74.107}, "visualize": {"avg_ms": 3.53, "count": 14, "max_ms": 3.864, "p95_ms": 3.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.193, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11009}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10342.191, "limit_mb": 8796093022207.996}, "detail": {"actual_bytes": 0, "expected_bytes": 2764800}, "event_type": "invalid_frame_read_size", "ffmpeg": {"error_count": 144, "h264_decode_error_count": 149, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:26:15+09:00", "last_error_summary": "[h264 @ 0x56221ec4b700] error while decoding MB 8 50, bytestream -5 [in#0/rtsp @ 0x56221e5e9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56221e5e9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:20:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 330.389, "last_started_at": "2026-07-01T00:20:44+09:00", "pid": 68761, "process": {"exists": false, "pid": 68761}, "restart_count": 143, "restart_count_per_hour": 7, "rtsp_timeout_count": 232, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 144}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.355, "sample_count": 2543, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1321112, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:26:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-01T00:26:15+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-01T00:26:14+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-01T00:26:14.960043+09:00", "mtime_epoch": 1782833174.960043, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105063}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:26:15+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1683.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.756, "pushed": 10.756, "read": 10.756}, "hints": ["ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 19.965, "count": 96, "max_ms": 25.526, "p95_ms": 23.319}, "gstreamer_push": {"avg_ms": 1.271, "count": 96, "max_ms": 1.821, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.814, "count": 96, "max_ms": 50.976, "p95_ms": 46.464}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.886, "count": 96, "max_ms": 58.484, "p95_ms": 54.037}, "yolo11_640_fire_11class": {"avg_ms": 46.991, "count": 96, "max_ms": 51.303, "p95_ms": 49.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.567, "count": 96, "max_ms": 51.835, "p95_ms": 49.934}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.363, "count": 96, "max_ms": 87.051, "p95_ms": 83.363}, "read_interval": {"avg_ms": 405.911, "count": 96, "max_ms": 30136.333, "p95_ms": 97.148}, "triton_wait": {"avg_ms": 50.92, "count": 96, "max_ms": 58.499, "p95_ms": 54.049}, "visualize": {"avg_ms": 3.66, "count": 96, "max_ms": 5.57, "p95_ms": 5.193}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 8.925, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11009}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10342.105, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 144, "h264_decode_error_count": 149, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:26:15+09:00", "last_error_summary": "[h264 @ 0x56221ec4b700] error while decoding MB 8 50, bytestream -5 [in#0/rtsp @ 0x56221e5e9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56221e5e9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:20:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 330.406, "last_started_at": "2026-07-01T00:20:44+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 143, "restart_count_per_hour": 7, "rtsp_timeout_count": 232, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 144}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.355, "sample_count": 2542, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1321112, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:26:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.034, "last_frame_pushed_at": "2026-07-01T00:26:15+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-01T00:26:14+09:00", "mjpeg": {"age_sec": 0.019, "exists": true, "mtime": "2026-07-01T00:26:15.048045+09:00", "mtime_epoch": 1782833175.048045, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105266}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:26:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1683.434, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.736, "pushed": 10.736, "read": 10.736}, "hints": ["ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 19.965, "count": 96, "max_ms": 25.526, "p95_ms": 23.319}, "gstreamer_push": {"avg_ms": 1.271, "count": 96, "max_ms": 1.821, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.814, "count": 96, "max_ms": 50.976, "p95_ms": 46.464}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.886, "count": 96, "max_ms": 58.484, "p95_ms": 54.037}, "yolo11_640_fire_11class": {"avg_ms": 46.991, "count": 96, "max_ms": 51.303, "p95_ms": 49.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.567, "count": 96, "max_ms": 51.835, "p95_ms": 49.934}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.363, "count": 96, "max_ms": 87.051, "p95_ms": 83.363}, "read_interval": {"avg_ms": 405.911, "count": 96, "max_ms": 30136.333, "p95_ms": 97.148}, "triton_wait": {"avg_ms": 50.92, "count": 96, "max_ms": 58.499, "p95_ms": 54.049}, "visualize": {"avg_ms": 3.66, "count": 96, "max_ms": 5.57, "p95_ms": 5.193}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 8.942, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11009}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10362.953, "limit_mb": 8796093022207.996}, "detail": {"pid": 68913, "restart_count": 144, "start_count": 145}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 144, "h264_decode_error_count": 149, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:26:15+09:00", "last_error_summary": "[h264 @ 0x56221ec4b700] error while decoding MB 8 50, bytestream -5 [in#0/rtsp @ 0x56221e5e9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56221e5e9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:20:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:26:17+09:00", "pid": 68913, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.211, "name": "ffmpeg", "pid": 68913, "rss_mb": 57.211, "state": "S (sleeping)", "threads": 1}, "restart_count": 144, "restart_count_per_hour": 8, "rtsp_timeout_count": 232, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 145}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.339, "sample_count": 2504, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1321112, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:26:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.512, "last_frame_pushed_at": "2026-07-01T00:26:15+09:00", "last_frame_read_age_sec": 4.588, "last_frame_read_at": "2026-07-01T00:26:14+09:00", "mjpeg": {"age_sec": 4.497, "exists": true, "mtime": "2026-07-01T00:26:15.048045+09:00", "mtime_epoch": 1782833175.048045, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105266}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:26:19+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1683.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.154, "pushed": 7.154, "read": 7.154}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 19.965, "count": 96, "max_ms": 25.526, "p95_ms": 23.319}, "gstreamer_push": {"avg_ms": 1.271, "count": 96, "max_ms": 1.821, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.814, "count": 96, "max_ms": 50.976, "p95_ms": 46.464}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.886, "count": 96, "max_ms": 58.484, "p95_ms": 54.037}, "yolo11_640_fire_11class": {"avg_ms": 46.991, "count": 96, "max_ms": 51.303, "p95_ms": 49.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.567, "count": 96, "max_ms": 51.835, "p95_ms": 49.934}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.363, "count": 96, "max_ms": 87.051, "p95_ms": 83.363}, "read_interval": {"avg_ms": 405.911, "count": 96, "max_ms": 30136.333, "p95_ms": 97.148}, "triton_wait": {"avg_ms": 50.92, "count": 96, "max_ms": 58.499, "p95_ms": 54.049}, "visualize": {"avg_ms": 3.66, "count": 96, "max_ms": 5.57, "p95_ms": 5.193}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 13.42, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10371.297, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561e6c3b9b00] error while decoding MB 98 12, bytestream -9 [in#0/rtsp @ 0x561e6c379e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561e6c379e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:37:22+09:00", "last_error_summary": "[h264 @ 0x561e6c3b9b00] error while decoding MB 98 12, bytestream -9 [in#0/rtsp @ 0x561e6c379e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561e6c379e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:20:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 664.618, "last_started_at": "2026-07-01T00:26:17+09:00", "pid": 68913, "process": {"exists": false, "pid": 68913}, "restart_count": 144, "restart_count_per_hour": 7, "rtsp_timeout_count": 234, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 145}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.0, "sample_count": 2643, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1327320, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:37:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T00:37:22+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T00:37:22+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-01T00:37:22.102236+09:00", "mtime_epoch": 1782833842.1022356, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106475}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:37:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1686.27, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.709, "pushed": 0.709, "read": 0.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.178, "count": 12, "max_ms": 23.161, "p95_ms": 23.161}, "gstreamer_push": {"avg_ms": 1.232, "count": 12, "max_ms": 1.327, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.111, "count": 12, "max_ms": 49.695, "p95_ms": 49.695}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.581, "count": 12, "max_ms": 50.508, "p95_ms": 50.508}, "yolo11_640_fire_11class": {"avg_ms": 47.233, "count": 12, "max_ms": 50.289, "p95_ms": 50.289}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.929, "count": 12, "max_ms": 50.269, "p95_ms": 50.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.791, "count": 12, "max_ms": 79.894, "p95_ms": 79.894}, "read_interval": {"avg_ms": 2369.105, "count": 13, "max_ms": 15852.456, "p95_ms": 15852.456}, "triton_wait": {"avg_ms": 48.748, "count": 12, "max_ms": 50.52, "p95_ms": 50.52}, "visualize": {"avg_ms": 3.339, "count": 12, "max_ms": 3.486, "p95_ms": 3.486}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.93, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10371.176, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:37:22+09:00", "last_error_summary": "[h264 @ 0x561e6c3b9b00] error while decoding MB 98 12, bytestream -9 [in#0/rtsp @ 0x561e6c379e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561e6c379e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:37:22+09:00", "last_error_summary": "[h264 @ 0x561e6c3b9b00] error while decoding MB 98 12, bytestream -9 [in#0/rtsp @ 0x561e6c379e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561e6c379e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:37:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 664.641, "last_started_at": "2026-07-01T00:26:17+09:00", "pid": 68913, "process": {"exists": false, "pid": 68913}, "restart_count": 144, "restart_count_per_hour": 7, "rtsp_timeout_count": 234, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 145}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.0, "sample_count": 2643, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1327321, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:37:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T00:37:22+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-01T00:37:22+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-01T00:37:22.102236+09:00", "mtime_epoch": 1782833842.1022356, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106475}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:37:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1686.27, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.767, "pushed": 0.767, "read": 0.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.395, "count": 13, "max_ms": 23.161, "p95_ms": 23.161}, "gstreamer_push": {"avg_ms": 1.218, "count": 13, "max_ms": 1.327, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.065, "count": 13, "max_ms": 49.695, "p95_ms": 49.695}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.53, "count": 13, "max_ms": 50.508, "p95_ms": 50.508}, "yolo11_640_fire_11class": {"avg_ms": 47.208, "count": 13, "max_ms": 50.289, "p95_ms": 50.289}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.915, "count": 13, "max_ms": 50.269, "p95_ms": 50.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.033, "count": 13, "max_ms": 79.937, "p95_ms": 79.937}, "read_interval": {"avg_ms": 2369.105, "count": 13, "max_ms": 15852.456, "p95_ms": 15852.456}, "triton_wait": {"avg_ms": 48.672, "count": 13, "max_ms": 50.52, "p95_ms": 50.52}, "visualize": {"avg_ms": 3.447, "count": 13, "max_ms": 4.741, "p95_ms": 4.741}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.951, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10371.293, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:37:22+09:00", "last_error_summary": "[h264 @ 0x561e6c3b9b00] error while decoding MB 98 12, bytestream -9 [in#0/rtsp @ 0x561e6c379e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561e6c379e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:37:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 664.646, "last_started_at": "2026-07-01T00:26:17+09:00", "pid": 68913, "process": {"exists": false, "pid": 68913}, "restart_count": 144, "restart_count_per_hour": 7, "rtsp_timeout_count": 234, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 145}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.0, "sample_count": 2643, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1327321, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:37:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T00:37:22+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-01T00:37:22+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T00:37:22.102236+09:00", "mtime_epoch": 1782833842.1022356, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 106475}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:37:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1686.27, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.767, "pushed": 0.767, "read": 0.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.395, "count": 13, "max_ms": 23.161, "p95_ms": 23.161}, "gstreamer_push": {"avg_ms": 1.218, "count": 13, "max_ms": 1.327, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.065, "count": 13, "max_ms": 49.695, "p95_ms": 49.695}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.53, "count": 13, "max_ms": 50.508, "p95_ms": 50.508}, "yolo11_640_fire_11class": {"avg_ms": 47.208, "count": 13, "max_ms": 50.289, "p95_ms": 50.289}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.915, "count": 13, "max_ms": 50.269, "p95_ms": 50.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.033, "count": 13, "max_ms": 79.937, "p95_ms": 79.937}, "read_interval": {"avg_ms": 2369.105, "count": 13, "max_ms": 15852.456, "p95_ms": 15852.456}, "triton_wait": {"avg_ms": 48.672, "count": 13, "max_ms": 50.52, "p95_ms": 50.52}, "visualize": {"avg_ms": 3.447, "count": 13, "max_ms": 4.741, "p95_ms": 4.741}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.956, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10391.738, "limit_mb": 8796093022207.996}, "detail": {"pid": 69065, "restart_count": 145, "start_count": 146}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:37:22+09:00", "last_error_summary": "[h264 @ 0x561e6c3b9b00] error while decoding MB 98 12, bytestream -9 [in#0/rtsp @ 0x561e6c379e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561e6c379e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:37:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:37:24+09:00", "pid": 69065, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.141, "name": "ffmpeg", "pid": 69065, "rss_mb": 58.141, "state": "S (sleeping)", "threads": 1}, "restart_count": 145, "restart_count_per_hour": 8, "rtsp_timeout_count": 234, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 146}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.027, "sample_count": 2597, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1327321, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:37:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.36, "last_frame_pushed_at": "2026-07-01T00:37:22+09:00", "last_frame_read_age_sec": 4.442, "last_frame_read_at": "2026-07-01T00:37:22+09:00", "mjpeg": {"age_sec": 4.344, "exists": true, "mtime": "2026-07-01T00:37:22.194237+09:00", "mtime_epoch": 1782833842.1942375, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:37:26+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1686.27, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.61, "pushed": 0.61, "read": 0.61}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.395, "count": 13, "max_ms": 23.161, "p95_ms": 23.161}, "gstreamer_push": {"avg_ms": 1.218, "count": 13, "max_ms": 1.327, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.065, "count": 13, "max_ms": 49.695, "p95_ms": 49.695}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.53, "count": 13, "max_ms": 50.508, "p95_ms": 50.508}, "yolo11_640_fire_11class": {"avg_ms": 47.208, "count": 13, "max_ms": 50.289, "p95_ms": 50.289}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.915, "count": 13, "max_ms": 50.269, "p95_ms": 50.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.033, "count": 13, "max_ms": 79.937, "p95_ms": 79.937}, "read_interval": {"avg_ms": 2369.105, "count": 13, "max_ms": 15852.456, "p95_ms": 15852.456}, "triton_wait": {"avg_ms": 48.672, "count": 13, "max_ms": 50.52, "p95_ms": 50.52}, "visualize": {"avg_ms": 3.447, "count": 13, "max_ms": 4.741, "p95_ms": 4.741}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.304, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11073}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10378.715, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559931c6b2c0] error while decoding MB 39 42, bytestream -5 [in#0/rtsp @ 0x559931c4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559931c4ae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 151, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:40:38+09:00", "last_error_summary": "[h264 @ 0x559931c6b2c0] error while decoding MB 39 42, bytestream -5 [in#0/rtsp @ 0x559931c4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559931c4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:37:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.31, "last_started_at": "2026-07-01T00:37:24+09:00", "pid": 69065, "process": {"exists": false, "pid": 69065}, "restart_count": 145, "restart_count_per_hour": 8, "rtsp_timeout_count": 236, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 146}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.438, "sample_count": 2158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1328783, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:40:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.083, "last_frame_pushed_at": "2026-07-01T00:40:38+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T00:40:38+09:00", "mjpeg": {"age_sec": 0.069, "exists": true, "mtime": "2026-07-01T00:40:38.778351+09:00", "mtime_epoch": 1782834038.7783506, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101395}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:40:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1687.043, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.73, "pushed": 0.73, "read": 0.787}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.127, "count": 13, "max_ms": 24.725, "p95_ms": 24.725}, "gstreamer_push": {"avg_ms": 1.217, "count": 13, "max_ms": 1.272, "p95_ms": 1.272}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.494, "count": 13, "max_ms": 60.023, "p95_ms": 60.023}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.426, "count": 13, "max_ms": 63.871, "p95_ms": 63.871}, "yolo11_640_fire_11class": {"avg_ms": 48.383, "count": 13, "max_ms": 60.595, "p95_ms": 60.595}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.46, "count": 13, "max_ms": 69.064, "p95_ms": 69.064}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.068, "count": 13, "max_ms": 95.801, "p95_ms": 95.801}, "read_interval": {"avg_ms": 2330.182, "count": 14, "max_ms": 16621.962, "p95_ms": 16621.962}, "triton_wait": {"avg_ms": 50.721, "count": 13, "max_ms": 69.077, "p95_ms": 69.077}, "visualize": {"avg_ms": 3.507, "count": 13, "max_ms": 4.371, "p95_ms": 4.371}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.797, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11073}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10378.52, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:40:38+09:00", "last_error_summary": "[h264 @ 0x559931c6b2c0] error while decoding MB 39 42, bytestream -5 [in#0/rtsp @ 0x559931c4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559931c4ae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 151, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:40:38+09:00", "last_error_summary": "[h264 @ 0x559931c6b2c0] error while decoding MB 39 42, bytestream -5 [in#0/rtsp @ 0x559931c4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559931c4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:40:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.334, "last_started_at": "2026-07-01T00:37:24+09:00", "pid": 69065, "process": {"exists": false, "pid": 69065}, "restart_count": 145, "restart_count_per_hour": 8, "rtsp_timeout_count": 236, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 146}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.438, "sample_count": 2159, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1328784, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:40:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T00:40:38+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T00:40:38+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-01T00:40:38.778351+09:00", "mtime_epoch": 1782834038.7783506, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101395}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:40:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1687.043, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.786, "pushed": 0.786, "read": 0.786}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.326, "count": 14, "max_ms": 24.725, "p95_ms": 24.725}, "gstreamer_push": {"avg_ms": 1.229, "count": 14, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.363, "count": 14, "max_ms": 60.023, "p95_ms": 60.023}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.548, "count": 14, "max_ms": 63.871, "p95_ms": 63.871}, "yolo11_640_fire_11class": {"avg_ms": 48.261, "count": 14, "max_ms": 60.595, "p95_ms": 60.595}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.227, "count": 14, "max_ms": 69.064, "p95_ms": 69.064}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.513, "count": 14, "max_ms": 95.801, "p95_ms": 95.801}, "read_interval": {"avg_ms": 2330.182, "count": 14, "max_ms": 16621.962, "p95_ms": 16621.962}, "triton_wait": {"avg_ms": 50.752, "count": 14, "max_ms": 69.077, "p95_ms": 69.077}, "visualize": {"avg_ms": 3.69, "count": 14, "max_ms": 6.07, "p95_ms": 6.07}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.821, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11073}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10378.375, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 151, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:40:38+09:00", "last_error_summary": "[h264 @ 0x559931c6b2c0] error while decoding MB 39 42, bytestream -5 [in#0/rtsp @ 0x559931c4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559931c4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:40:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.342, "last_started_at": "2026-07-01T00:37:24+09:00", "pid": 69065, "process": {"exists": false, "pid": 69065}, "restart_count": 145, "restart_count_per_hour": 8, "rtsp_timeout_count": 236, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 146}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.438, "sample_count": 2159, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1328784, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:40:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T00:40:38+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T00:40:38+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-01T00:40:38.778351+09:00", "mtime_epoch": 1782834038.7783506, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101395}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:40:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1687.043, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.785, "pushed": 0.785, "read": 0.785}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.326, "count": 14, "max_ms": 24.725, "p95_ms": 24.725}, "gstreamer_push": {"avg_ms": 1.229, "count": 14, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.363, "count": 14, "max_ms": 60.023, "p95_ms": 60.023}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.548, "count": 14, "max_ms": 63.871, "p95_ms": 63.871}, "yolo11_640_fire_11class": {"avg_ms": 48.261, "count": 14, "max_ms": 60.595, "p95_ms": 60.595}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.227, "count": 14, "max_ms": 69.064, "p95_ms": 69.064}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.513, "count": 14, "max_ms": 95.801, "p95_ms": 95.801}, "read_interval": {"avg_ms": 2330.182, "count": 14, "max_ms": 16621.962, "p95_ms": 16621.962}, "triton_wait": {"avg_ms": 50.752, "count": 14, "max_ms": 69.077, "p95_ms": 69.077}, "visualize": {"avg_ms": 3.69, "count": 14, "max_ms": 6.07, "p95_ms": 6.07}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.829, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11073}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10553.637, "limit_mb": 8796093022207.996}, "detail": {"pid": 69217, "restart_count": 146, "start_count": 147}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 151, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:40:38+09:00", "last_error_summary": "[h264 @ 0x559931c6b2c0] error while decoding MB 39 42, bytestream -5 [in#0/rtsp @ 0x559931c4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559931c4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:40:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:40:41+09:00", "pid": 69217, "process": {"exists": true, "fd_count": 5, "hwm_mb": 199.926, "name": "ffmpeg", "pid": 69217, "rss_mb": 199.926, "state": "R (running)", "threads": 106}, "restart_count": 146, "restart_count_per_hour": 9, "rtsp_timeout_count": 236, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.465, "sample_count": 2115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1328784, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:40:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.265, "last_frame_pushed_at": "2026-07-01T00:40:38+09:00", "last_frame_read_age_sec": 4.354, "last_frame_read_at": "2026-07-01T00:40:38+09:00", "mjpeg": {"age_sec": 4.249, "exists": true, "mtime": "2026-07-01T00:40:38.882353+09:00", "mtime_epoch": 1782834038.8823526, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:40:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1687.043, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.634, "pushed": 0.634, "read": 0.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.326, "count": 14, "max_ms": 24.725, "p95_ms": 24.725}, "gstreamer_push": {"avg_ms": 1.229, "count": 14, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.363, "count": 14, "max_ms": 60.023, "p95_ms": 60.023}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.548, "count": 14, "max_ms": 63.871, "p95_ms": 63.871}, "yolo11_640_fire_11class": {"avg_ms": 48.261, "count": 14, "max_ms": 60.595, "p95_ms": 60.595}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.227, "count": 14, "max_ms": 69.064, "p95_ms": 69.064}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.513, "count": 14, "max_ms": 95.801, "p95_ms": 95.801}, "read_interval": {"avg_ms": 2330.182, "count": 14, "max_ms": 16621.962, "p95_ms": 16621.962}, "triton_wait": {"avg_ms": 50.752, "count": 14, "max_ms": 69.077, "p95_ms": 69.077}, "visualize": {"avg_ms": 3.69, "count": 14, "max_ms": 6.07, "p95_ms": 6.07}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.081, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9881}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10406.836, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b4c735a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b4c735a100] error while decoding MB 0 43, bytestream 239 [in#0/rtsp @ 0x55b4c7224e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4c7224e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 350, "h264_decode_error_count": 1118, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:42:18+09:00", "last_error_summary": "[h264 @ 0x55b4c735a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b4c735a100] error while decoding MB 0 43, bytestream 239 [in#0/rtsp @ 0x55b4c7224e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4c7224e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:20:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 1325.233, "last_started_at": "2026-07-01T00:20:13+09:00", "pid": 68654, "process": {"exists": false, "pid": 68654}, "restart_count": 383, "restart_count_per_hour": 3, "rtsp_timeout_count": 318, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 384}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.652, "sample_count": 2669, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1185770, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:42:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-01T00:42:18+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-01T00:42:18+09:00", "mjpeg": {"age_sec": 0.065, "exists": true, "mtime": "2026-07-01T00:42:18.215162+09:00", "mtime_epoch": 1782834138.2151618, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119260}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:42:18+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4929.773, "name": "celery", "pid": 267, "rss_mb": 4929.773, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.372, "pushed": 3.372, "read": 3.372}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.933, "count": 101, "max_ms": 24.778, "p95_ms": 24.456}, "gstreamer_push": {"avg_ms": 1.207, "count": 101, "max_ms": 1.403, "p95_ms": 1.331}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.222, "count": 101, "max_ms": 43.256, "p95_ms": 40.083}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.726, "count": 101, "max_ms": 67.789, "p95_ms": 64.471}, "yolo11_640_fire_11class": {"avg_ms": 55.721, "count": 101, "max_ms": 65.27, "p95_ms": 63.966}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.79, "count": 101, "max_ms": 74.303, "p95_ms": 73.473}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.747, "count": 101, "max_ms": 102.603, "p95_ms": 101.318}, "read_interval": {"avg_ms": 296.791, "count": 101, "max_ms": 19309.488, "p95_ms": 117.81}, "triton_wait": {"avg_ms": 62.819, "count": 101, "max_ms": 74.315, "p95_ms": 73.485}, "visualize": {"avg_ms": 3.363, "count": 101, "max_ms": 3.997, "p95_ms": 3.579}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9881}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10406.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:42:18+09:00", "last_error_summary": "[h264 @ 0x55b4c735a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b4c735a100] error while decoding MB 0 43, bytestream 239 [in#0/rtsp @ 0x55b4c7224e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4c7224e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 350, "h264_decode_error_count": 1118, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:42:18+09:00", "last_error_summary": "[h264 @ 0x55b4c735a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b4c735a100] error while decoding MB 0 43, bytestream 239 [in#0/rtsp @ 0x55b4c7224e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4c7224e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:42:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 1325.258, "last_started_at": "2026-07-01T00:20:13+09:00", "pid": 68654, "process": {"exists": false, "pid": 68654}, "restart_count": 383, "restart_count_per_hour": 3, "rtsp_timeout_count": 318, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 384}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.652, "sample_count": 2670, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1185771, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:42:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T00:42:18+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T00:42:18+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-01T00:42:18.215162+09:00", "mtime_epoch": 1782834138.2151618, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119260}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:42:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4929.773, "name": "celery", "pid": 267, "rss_mb": 4929.773, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.403, "pushed": 3.403, "read": 3.37}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.938, "count": 102, "max_ms": 24.778, "p95_ms": 24.456}, "gstreamer_push": {"avg_ms": 1.209, "count": 102, "max_ms": 1.403, "p95_ms": 1.332}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.194, "count": 102, "max_ms": 43.256, "p95_ms": 40.083}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.616, "count": 102, "max_ms": 67.789, "p95_ms": 64.471}, "yolo11_640_fire_11class": {"avg_ms": 55.609, "count": 102, "max_ms": 65.27, "p95_ms": 63.966}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.626, "count": 102, "max_ms": 74.303, "p95_ms": 73.473}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.65, "count": 102, "max_ms": 102.603, "p95_ms": 101.318}, "read_interval": {"avg_ms": 296.791, "count": 101, "max_ms": 19309.488, "p95_ms": 117.81}, "triton_wait": {"avg_ms": 62.655, "count": 102, "max_ms": 74.315, "p95_ms": 73.485}, "visualize": {"avg_ms": 3.412, "count": 102, "max_ms": 8.355, "p95_ms": 3.58}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9881}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10406.359, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 350, "h264_decode_error_count": 1118, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:42:18+09:00", "last_error_summary": "[h264 @ 0x55b4c735a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b4c735a100] error while decoding MB 0 43, bytestream 239 [in#0/rtsp @ 0x55b4c7224e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4c7224e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:42:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 1325.266, "last_started_at": "2026-07-01T00:20:13+09:00", "pid": 68654, "process": {"exists": false, "pid": 68654}, "restart_count": 383, "restart_count_per_hour": 3, "rtsp_timeout_count": 318, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 384}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.652, "sample_count": 2670, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1185771, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:42:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-01T00:42:18+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T00:42:18+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-01T00:42:18.215162+09:00", "mtime_epoch": 1782834138.2151618, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119260}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:42:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4929.773, "name": "celery", "pid": 267, "rss_mb": 4929.773, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.402, "pushed": 3.402, "read": 3.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.938, "count": 102, "max_ms": 24.778, "p95_ms": 24.456}, "gstreamer_push": {"avg_ms": 1.209, "count": 102, "max_ms": 1.403, "p95_ms": 1.332}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.194, "count": 102, "max_ms": 43.256, "p95_ms": 40.083}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.616, "count": 102, "max_ms": 67.789, "p95_ms": 64.471}, "yolo11_640_fire_11class": {"avg_ms": 55.609, "count": 102, "max_ms": 65.27, "p95_ms": 63.966}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.626, "count": 102, "max_ms": 74.303, "p95_ms": 73.473}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.65, "count": 102, "max_ms": 102.603, "p95_ms": 101.318}, "read_interval": {"avg_ms": 296.791, "count": 101, "max_ms": 19309.488, "p95_ms": 117.81}, "triton_wait": {"avg_ms": 62.655, "count": 102, "max_ms": 74.315, "p95_ms": 73.485}, "visualize": {"avg_ms": 3.412, "count": 102, "max_ms": 8.355, "p95_ms": 3.58}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9881}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10514.672, "limit_mb": 8796093022207.996}, "detail": {"pid": 69369, "restart_count": 384, "start_count": 385}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 350, "h264_decode_error_count": 1118, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:42:18+09:00", "last_error_summary": "[h264 @ 0x55b4c735a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b4c735a100] error while decoding MB 0 43, bytestream 239 [in#0/rtsp @ 0x55b4c7224e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4c7224e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:42:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:42:21+09:00", "pid": 69369, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.75, "name": "ffmpeg", "pid": 69369, "rss_mb": 138.75, "state": "S (sleeping)", "threads": 106}, "restart_count": 384, "restart_count_per_hour": 4, "rtsp_timeout_count": 318, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 385}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.671, "sample_count": 2616, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1185771, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:42:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.937, "last_frame_pushed_at": "2026-07-01T00:42:18+09:00", "last_frame_read_age_sec": 5.021, "last_frame_read_at": "2026-07-01T00:42:18+09:00", "mjpeg": {"age_sec": 4.921, "exists": true, "mtime": "2026-07-01T00:42:18.311161+09:00", "mtime_epoch": 1782834138.311161, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122319}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:42:23+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4929.773, "name": "celery", "pid": 267, "rss_mb": 4929.773, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.967, "pushed": 1.967, "read": 1.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.641, "count": 58, "max_ms": 24.778, "p95_ms": 24.625}, "gstreamer_push": {"avg_ms": 1.223, "count": 59, "max_ms": 1.403, "p95_ms": 1.374}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.202, "count": 58, "max_ms": 43.256, "p95_ms": 40.028}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.693, "count": 58, "max_ms": 64.471, "p95_ms": 62.661}, "yolo11_640_fire_11class": {"avg_ms": 51.841, "count": 58, "max_ms": 63.966, "p95_ms": 62.251}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.909, "count": 58, "max_ms": 73.972, "p95_ms": 72.018}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.843, "count": 59, "max_ms": 99.939, "p95_ms": 99.293}, "read_interval": {"avg_ms": 432.303, "count": 58, "max_ms": 19309.488, "p95_ms": 114.808}, "triton_wait": {"avg_ms": 56.951, "count": 58, "max_ms": 73.983, "p95_ms": 72.031}, "visualize": {"avg_ms": 3.463, "count": 58, "max_ms": 8.355, "p95_ms": 3.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10410.453, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563a04175c00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04175c00] error while decoding MB 0 3, bytestream 4173 [h264 @ 0x563a04b6ee80] error while decoding MB 13 21, bytestream -5 [h264 @ 0x563a04189dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04189dc0] error while decoding MB 0 30, bytestream 52473 [h264 @ 0x563a0404cec0] left block unavailable for requested intra mode [h264 @ 0x563a0404cec0] error while decoding MB 0 33, bytestrea", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 351, "h264_decode_error_count": 1126, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:47:16+09:00", "last_error_summary": "[h264 @ 0x563a04175c00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04175c00] error while decoding MB 0 3, bytestream 4173 [h264 @ 0x563a04b6ee80] error while decoding MB 13 21, bytestream -5 [h264 @ 0x563a04189dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04189dc0] error while decoding MB 0 30, bytestream 52473 [h264 @ 0x563a0404cec0] left block unavailable for requested intra mode [h264 @ 0x563a0404cec0] error while decoding MB 0 33, bytestrea", "last_exit_at": "2026-07-01T00:42:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 295.263, "last_started_at": "2026-07-01T00:42:21+09:00", "pid": 69369, "process": {"exists": false, "pid": 69369}, "restart_count": 384, "restart_count_per_hour": 4, "rtsp_timeout_count": 320, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 385}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.872, "sample_count": 2562, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1188313, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:47:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T00:47:16+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T00:47:16+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-01T00:47:16.434436+09:00", "mtime_epoch": 1782834436.4344358, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127824}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:47:16+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4931.062, "name": "celery", "pid": 267, "rss_mb": 4931.062, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.902, "pushed": 3.902, "read": 3.902}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.878, "count": 117, "max_ms": 28.797, "p95_ms": 24.724}, "gstreamer_push": {"avg_ms": 1.241, "count": 117, "max_ms": 1.477, "p95_ms": 1.349}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.71, "count": 117, "max_ms": 35.153, "p95_ms": 33.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.673, "count": 117, "max_ms": 49.739, "p95_ms": 46.645}, "yolo11_640_fire_11class": {"avg_ms": 43.939, "count": 117, "max_ms": 48.852, "p95_ms": 45.551}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.402, "count": 117, "max_ms": 53.835, "p95_ms": 46.876}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.382, "count": 117, "max_ms": 83.957, "p95_ms": 79.931}, "read_interval": {"avg_ms": 256.342, "count": 117, "max_ms": 19360.337, "p95_ms": 96.261}, "triton_wait": {"avg_ms": 45.462, "count": 117, "max_ms": 53.847, "p95_ms": 47.371}, "visualize": {"avg_ms": 3.398, "count": 117, "max_ms": 4.468, "p95_ms": 3.642}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10409.66, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:47:16+09:00", "last_error_summary": "[h264 @ 0x563a04175c00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04175c00] error while decoding MB 0 3, bytestream 4173 [h264 @ 0x563a04b6ee80] error while decoding MB 13 21, bytestream -5 [h264 @ 0x563a04189dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04189dc0] error while decoding MB 0 30, bytestream 52473 [h264 @ 0x563a0404cec0] left block unavailable for requested intra mode [h264 @ 0x563a0404cec0] error while decoding MB 0 33, bytestrea", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 351, "h264_decode_error_count": 1126, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:47:16+09:00", "last_error_summary": "[h264 @ 0x563a04175c00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04175c00] error while decoding MB 0 3, bytestream 4173 [h264 @ 0x563a04b6ee80] error while decoding MB 13 21, bytestream -5 [h264 @ 0x563a04189dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04189dc0] error while decoding MB 0 30, bytestream 52473 [h264 @ 0x563a0404cec0] left block unavailable for requested intra mode [h264 @ 0x563a0404cec0] error while decoding MB 0 33, bytestrea", "last_exit_at": "2026-07-01T00:47:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 295.292, "last_started_at": "2026-07-01T00:42:21+09:00", "pid": 69369, "process": {"exists": false, "pid": 69369}, "restart_count": 384, "restart_count_per_hour": 4, "rtsp_timeout_count": 320, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 385}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.872, "sample_count": 2563, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1188314, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:47:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T00:47:16+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-01T00:47:16+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-01T00:47:16.434436+09:00", "mtime_epoch": 1782834436.4344358, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127824}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:47:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4931.062, "name": "celery", "pid": 267, "rss_mb": 4931.062, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.9, "pushed": 3.9, "read": 3.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.917, "count": 117, "max_ms": 28.797, "p95_ms": 24.803}, "gstreamer_push": {"avg_ms": 1.243, "count": 117, "max_ms": 1.477, "p95_ms": 1.351}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.731, "count": 117, "max_ms": 35.153, "p95_ms": 33.406}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.669, "count": 117, "max_ms": 49.739, "p95_ms": 46.645}, "yolo11_640_fire_11class": {"avg_ms": 43.939, "count": 117, "max_ms": 48.852, "p95_ms": 45.551}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.407, "count": 117, "max_ms": 53.835, "p95_ms": 46.876}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.445, "count": 117, "max_ms": 84.686, "p95_ms": 80.372}, "read_interval": {"avg_ms": 256.342, "count": 117, "max_ms": 19360.337, "p95_ms": 96.261}, "triton_wait": {"avg_ms": 45.463, "count": 117, "max_ms": 53.847, "p95_ms": 47.371}, "visualize": {"avg_ms": 3.417, "count": 117, "max_ms": 5.562, "p95_ms": 3.802}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10409.652, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 351, "h264_decode_error_count": 1126, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:47:16+09:00", "last_error_summary": "[h264 @ 0x563a04175c00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04175c00] error while decoding MB 0 3, bytestream 4173 [h264 @ 0x563a04b6ee80] error while decoding MB 13 21, bytestream -5 [h264 @ 0x563a04189dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04189dc0] error while decoding MB 0 30, bytestream 52473 [h264 @ 0x563a0404cec0] left block unavailable for requested intra mode [h264 @ 0x563a0404cec0] error while decoding MB 0 33, bytestrea", "last_exit_at": "2026-07-01T00:47:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 295.3, "last_started_at": "2026-07-01T00:42:21+09:00", "pid": 69369, "process": {"exists": false, "pid": 69369}, "restart_count": 384, "restart_count_per_hour": 4, "rtsp_timeout_count": 320, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 385}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.872, "sample_count": 2563, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1188314, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:47:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-01T00:47:16+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-01T00:47:16+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-01T00:47:16.434436+09:00", "mtime_epoch": 1782834436.4344358, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127824}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:47:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4931.062, "name": "celery", "pid": 267, "rss_mb": 4931.062, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.909, "pushed": 3.909, "read": 3.876}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.917, "count": 117, "max_ms": 28.797, "p95_ms": 24.803}, "gstreamer_push": {"avg_ms": 1.243, "count": 117, "max_ms": 1.477, "p95_ms": 1.351}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.731, "count": 117, "max_ms": 35.153, "p95_ms": 33.406}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.669, "count": 117, "max_ms": 49.739, "p95_ms": 46.645}, "yolo11_640_fire_11class": {"avg_ms": 43.939, "count": 117, "max_ms": 48.852, "p95_ms": 45.551}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.407, "count": 117, "max_ms": 53.835, "p95_ms": 46.876}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.445, "count": 117, "max_ms": 84.686, "p95_ms": 80.372}, "read_interval": {"avg_ms": 257.755, "count": 116, "max_ms": 19360.337, "p95_ms": 96.261}, "triton_wait": {"avg_ms": 45.463, "count": 117, "max_ms": 53.847, "p95_ms": 47.371}, "visualize": {"avg_ms": 3.417, "count": 117, "max_ms": 5.562, "p95_ms": 3.802}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10520.383, "limit_mb": 8796093022207.996}, "detail": {"pid": 69476, "restart_count": 385, "start_count": 386}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 351, "h264_decode_error_count": 1126, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:47:16+09:00", "last_error_summary": "[h264 @ 0x563a04175c00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04175c00] error while decoding MB 0 3, bytestream 4173 [h264 @ 0x563a04b6ee80] error while decoding MB 13 21, bytestream -5 [h264 @ 0x563a04189dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563a04189dc0] error while decoding MB 0 30, bytestream 52473 [h264 @ 0x563a0404cec0] left block unavailable for requested intra mode [h264 @ 0x563a0404cec0] error while decoding MB 0 33, bytestrea", "last_exit_at": "2026-07-01T00:47:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:47:19+09:00", "pid": 69476, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.562, "name": "ffmpeg", "pid": 69476, "rss_mb": 141.562, "state": "S (sleeping)", "threads": 106}, "restart_count": 385, "restart_count_per_hour": 5, "rtsp_timeout_count": 320, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 386}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.882, "sample_count": 2543, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1188314, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:47:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 5.008, "last_frame_pushed_at": "2026-07-01T00:47:16+09:00", "last_frame_read_age_sec": 5.095, "last_frame_read_at": "2026-07-01T00:47:16+09:00", "mjpeg": {"age_sec": 4.991, "exists": true, "mtime": "2026-07-01T00:47:16.530436+09:00", "mtime_epoch": 1782834436.530436, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123798}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T00:47:21+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4931.062, "name": "celery", "pid": 267, "rss_mb": 4931.062, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.102, "pushed": 2.102, "read": 2.069}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.725, "count": 63, "max_ms": 28.426, "p95_ms": 24.426}, "gstreamer_push": {"avg_ms": 1.243, "count": 63, "max_ms": 1.477, "p95_ms": 1.356}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.402, "count": 63, "max_ms": 35.153, "p95_ms": 32.565}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.917, "count": 63, "max_ms": 49.739, "p95_ms": 46.649}, "yolo11_640_fire_11class": {"avg_ms": 44.126, "count": 63, "max_ms": 46.203, "p95_ms": 45.73}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.635, "count": 63, "max_ms": 53.835, "p95_ms": 47.358}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.462, "count": 63, "max_ms": 84.686, "p95_ms": 79.931}, "read_interval": {"avg_ms": 402.275, "count": 62, "max_ms": 19360.337, "p95_ms": 95.242}, "triton_wait": {"avg_ms": 45.681, "count": 63, "max_ms": 53.847, "p95_ms": 47.565}, "visualize": {"avg_ms": 3.42, "count": 63, "max_ms": 5.562, "p95_ms": 3.802}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11123}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10361.645, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55eb15643200] missing picture in access unit with size 18 [h264 @ 0x55eb1564fa40] no frame! [in#0/rtsp @ 0x55eb1563ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55eb1563ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55eb15643840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 152, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:51:32+09:00", "last_error_summary": "[NULL @ 0x55eb15643200] missing picture in access unit with size 18 [h264 @ 0x55eb1564fa40] no frame! [in#0/rtsp @ 0x55eb1563ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55eb1563ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55eb15643840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T00:40:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 651.701, "last_started_at": "2026-07-01T00:40:41+09:00", "pid": 69217, "process": {"exists": false, "pid": 69217}, "restart_count": 146, "restart_count_per_hour": 8, "rtsp_timeout_count": 238, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.442, "sample_count": 2793, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1334858, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:51:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T00:51:32+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-01T00:51:32+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T00:51:32.771927+09:00", "mtime_epoch": 1782834692.7719269, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91834}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:51:32+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1689.879, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 44, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.467, "pushed": 1.467, "read": 1.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.934, "count": 43, "max_ms": 23.157, "p95_ms": 22.496}, "gstreamer_push": {"avg_ms": 1.251, "count": 44, "max_ms": 1.393, "p95_ms": 1.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.501, "count": 43, "max_ms": 67.427, "p95_ms": 56.827}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.113, "count": 43, "max_ms": 69.86, "p95_ms": 69.005}, "yolo11_640_fire_11class": {"avg_ms": 60.051, "count": 43, "max_ms": 68.831, "p95_ms": 68.821}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.309, "count": 43, "max_ms": 79.528, "p95_ms": 79.244}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.702, "count": 44, "max_ms": 106.6, "p95_ms": 106.024}, "read_interval": {"avg_ms": 682.798, "count": 44, "max_ms": 16231.728, "p95_ms": 123.222}, "triton_wait": {"avg_ms": 67.321, "count": 43, "max_ms": 79.541, "p95_ms": 79.257}, "visualize": {"avg_ms": 3.437, "count": 43, "max_ms": 3.953, "p95_ms": 3.575}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11123}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10361.145, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:51:32+09:00", "last_error_summary": "[NULL @ 0x55eb15643200] missing picture in access unit with size 18 [h264 @ 0x55eb1564fa40] no frame! [in#0/rtsp @ 0x55eb1563ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55eb1563ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55eb15643840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 152, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:51:32+09:00", "last_error_summary": "[NULL @ 0x55eb15643200] missing picture in access unit with size 18 [h264 @ 0x55eb1564fa40] no frame! [in#0/rtsp @ 0x55eb1563ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55eb1563ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55eb15643840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T00:51:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 651.754, "last_started_at": "2026-07-01T00:40:41+09:00", "pid": 69217, "process": {"exists": false, "pid": 69217}, "restart_count": 146, "restart_count_per_hour": 8, "rtsp_timeout_count": 238, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.477, "sample_count": 2793, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1334859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:51:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T00:51:32+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-01T00:51:32+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-01T00:51:32.771927+09:00", "mtime_epoch": 1782834692.7719269, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91834}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:51:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1689.879, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.47, "pushed": 1.47, "read": 1.437}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.934, "count": 44, "max_ms": 23.157, "p95_ms": 22.496}, "gstreamer_push": {"avg_ms": 1.252, "count": 44, "max_ms": 1.393, "p95_ms": 1.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.725, "count": 44, "max_ms": 67.427, "p95_ms": 56.827}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.36, "count": 44, "max_ms": 71.987, "p95_ms": 69.372}, "yolo11_640_fire_11class": {"avg_ms": 60.281, "count": 44, "max_ms": 70.165, "p95_ms": 68.828}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.59, "count": 44, "max_ms": 79.691, "p95_ms": 79.249}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.724, "count": 44, "max_ms": 106.608, "p95_ms": 106.047}, "read_interval": {"avg_ms": 695.873, "count": 43, "max_ms": 16231.728, "p95_ms": 123.222}, "triton_wait": {"avg_ms": 67.603, "count": 44, "max_ms": 79.703, "p95_ms": 79.262}, "visualize": {"avg_ms": 3.436, "count": 44, "max_ms": 3.953, "p95_ms": 3.575}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11123}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10361.117, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 152, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:51:32+09:00", "last_error_summary": "[NULL @ 0x55eb15643200] missing picture in access unit with size 18 [h264 @ 0x55eb1564fa40] no frame! [in#0/rtsp @ 0x55eb1563ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55eb1563ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55eb15643840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T00:51:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 651.761, "last_started_at": "2026-07-01T00:40:41+09:00", "pid": 69217, "process": {"exists": false, "pid": 69217}, "restart_count": 146, "restart_count_per_hour": 8, "rtsp_timeout_count": 238, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.477, "sample_count": 2793, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1334859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:51:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T00:51:32+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-01T00:51:32+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-01T00:51:32.771927+09:00", "mtime_epoch": 1782834692.7719269, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91834}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:51:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1689.879, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.47, "pushed": 1.47, "read": 1.436}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.934, "count": 44, "max_ms": 23.157, "p95_ms": 22.496}, "gstreamer_push": {"avg_ms": 1.252, "count": 44, "max_ms": 1.393, "p95_ms": 1.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.725, "count": 44, "max_ms": 67.427, "p95_ms": 56.827}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.36, "count": 44, "max_ms": 71.987, "p95_ms": 69.372}, "yolo11_640_fire_11class": {"avg_ms": 60.281, "count": 44, "max_ms": 70.165, "p95_ms": 68.828}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.59, "count": 44, "max_ms": 79.691, "p95_ms": 79.249}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.724, "count": 44, "max_ms": 106.608, "p95_ms": 106.047}, "read_interval": {"avg_ms": 695.873, "count": 43, "max_ms": 16231.728, "p95_ms": 123.222}, "triton_wait": {"avg_ms": 67.603, "count": 44, "max_ms": 79.703, "p95_ms": 79.262}, "visualize": {"avg_ms": 3.436, "count": 44, "max_ms": 3.953, "p95_ms": 3.575}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11123}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10540.945, "limit_mb": 8796093022207.996}, "detail": {"pid": 69583, "restart_count": 147, "start_count": 148}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 152, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:51:32+09:00", "last_error_summary": "[NULL @ 0x55eb15643200] missing picture in access unit with size 18 [h264 @ 0x55eb1564fa40] no frame! [in#0/rtsp @ 0x55eb1563ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55eb1563ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55eb15643840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T00:51:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:51:35+09:00", "pid": 69583, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.113, "name": "ffmpeg", "pid": 69583, "rss_mb": 209.113, "state": "S (sleeping)", "threads": 106}, "restart_count": 147, "restart_count_per_hour": 9, "rtsp_timeout_count": 238, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.499, "sample_count": 2743, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1334859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:51:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.63, "last_frame_pushed_at": "2026-07-01T00:51:32+09:00", "last_frame_read_age_sec": 4.738, "last_frame_read_at": "2026-07-01T00:51:32+09:00", "mjpeg": {"age_sec": 4.613, "exists": true, "mtime": "2026-07-01T00:51:32.895928+09:00", "mtime_epoch": 1782834692.895928, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92423}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:51:37+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1689.879, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.589, "pushed": 0.589, "read": 0.589}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.975, "count": 13, "max_ms": 22.82, "p95_ms": 22.82}, "gstreamer_push": {"avg_ms": 1.273, "count": 13, "max_ms": 1.393, "p95_ms": 1.393}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.381, "count": 13, "max_ms": 56.381, "p95_ms": 56.381}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.078, "count": 13, "max_ms": 71.987, "p95_ms": 71.987}, "yolo11_640_fire_11class": {"avg_ms": 56.19, "count": 13, "max_ms": 70.165, "p95_ms": 70.165}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.637, "count": 13, "max_ms": 79.691, "p95_ms": 79.691}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.915, "count": 13, "max_ms": 106.608, "p95_ms": 106.608}, "read_interval": {"avg_ms": 2038.852, "count": 13, "max_ms": 16231.728, "p95_ms": 16231.728}, "triton_wait": {"avg_ms": 60.648, "count": 13, "max_ms": 79.703, "p95_ms": 79.703}, "visualize": {"avg_ms": 3.42, "count": 13, "max_ms": 3.55, "p95_ms": 3.55}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.081, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10365.715, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d00beb5580] error while decoding MB 54 24, bytestream -11 [in#0/rtsp @ 0x55d00be73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d00be73e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 153, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:53:24+09:00", "last_error_summary": "[h264 @ 0x55d00beb5580] error while decoding MB 54 24, bytestream -11 [in#0/rtsp @ 0x55d00be73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d00be73e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:51:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.23, "last_started_at": "2026-07-01T00:51:35+09:00", "pid": 69583, "process": {"exists": false, "pid": 69583}, "restart_count": 147, "restart_count_per_hour": 9, "rtsp_timeout_count": 240, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.5, "sample_count": 2342, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1335497, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:53:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T00:53:24+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T00:53:24+09:00", "mjpeg": {"age_sec": 0.065, "exists": true, "mtime": "2026-07-01T00:53:24.672970+09:00", "mtime_epoch": 1782834804.6729696, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93255}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:53:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1690.137, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.657, "pushed": 0.657, "read": 0.711}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.197, "count": 12, "max_ms": 21.712, "p95_ms": 21.712}, "gstreamer_push": {"avg_ms": 1.231, "count": 12, "max_ms": 1.31, "p95_ms": 1.31}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.871, "count": 12, "max_ms": 50.007, "p95_ms": 50.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.362, "count": 12, "max_ms": 70.357, "p95_ms": 70.357}, "yolo11_640_fire_11class": {"avg_ms": 49.202, "count": 12, "max_ms": 65.824, "p95_ms": 65.824}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.538, "count": 12, "max_ms": 74.836, "p95_ms": 74.836}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.17, "count": 12, "max_ms": 101.438, "p95_ms": 101.438}, "read_interval": {"avg_ms": 2796.354, "count": 13, "max_ms": 18145.507, "p95_ms": 18145.507}, "triton_wait": {"avg_ms": 50.845, "count": 12, "max_ms": 74.848, "p95_ms": 74.848}, "visualize": {"avg_ms": 3.427, "count": 12, "max_ms": 4.375, "p95_ms": 4.375}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.274, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10365.41, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:53:24+09:00", "last_error_summary": "[h264 @ 0x55d00beb5580] error while decoding MB 54 24, bytestream -11 [in#0/rtsp @ 0x55d00be73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d00be73e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 153, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:53:24+09:00", "last_error_summary": "[h264 @ 0x55d00beb5580] error while decoding MB 54 24, bytestream -11 [in#0/rtsp @ 0x55d00be73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d00be73e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:53:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.256, "last_started_at": "2026-07-01T00:51:35+09:00", "pid": 69583, "process": {"exists": false, "pid": 69583}, "restart_count": 147, "restart_count_per_hour": 9, "rtsp_timeout_count": 240, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.5, "sample_count": 2343, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1335498, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:53:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T00:53:24+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-01T00:53:24+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T00:53:24.672970+09:00", "mtime_epoch": 1782834804.6729696, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93255}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:53:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1690.137, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.71, "pushed": 0.71, "read": 0.71}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.46, "count": 13, "max_ms": 23.616, "p95_ms": 23.616}, "gstreamer_push": {"avg_ms": 1.25, "count": 13, "max_ms": 1.472, "p95_ms": 1.472}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.713, "count": 13, "max_ms": 50.007, "p95_ms": 50.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.19, "count": 13, "max_ms": 70.357, "p95_ms": 70.357}, "yolo11_640_fire_11class": {"avg_ms": 48.949, "count": 13, "max_ms": 65.824, "p95_ms": 65.824}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.316, "count": 13, "max_ms": 74.836, "p95_ms": 74.836}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.368, "count": 13, "max_ms": 101.438, "p95_ms": 101.438}, "read_interval": {"avg_ms": 2796.354, "count": 13, "max_ms": 18145.507, "p95_ms": 18145.507}, "triton_wait": {"avg_ms": 50.601, "count": 13, "max_ms": 74.848, "p95_ms": 74.848}, "visualize": {"avg_ms": 3.549, "count": 13, "max_ms": 5.018, "p95_ms": 5.018}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.3, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10365.48, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 153, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:53:24+09:00", "last_error_summary": "[h264 @ 0x55d00beb5580] error while decoding MB 54 24, bytestream -11 [in#0/rtsp @ 0x55d00be73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d00be73e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:53:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.263, "last_started_at": "2026-07-01T00:51:35+09:00", "pid": 69583, "process": {"exists": false, "pid": 69583}, "restart_count": 147, "restart_count_per_hour": 9, "rtsp_timeout_count": 240, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.5, "sample_count": 2343, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1335498, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:53:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T00:53:24+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-01T00:53:24+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-01T00:53:24.672970+09:00", "mtime_epoch": 1782834804.6729696, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93255}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:53:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1690.137, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.71, "pushed": 0.71, "read": 0.71}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.46, "count": 13, "max_ms": 23.616, "p95_ms": 23.616}, "gstreamer_push": {"avg_ms": 1.25, "count": 13, "max_ms": 1.472, "p95_ms": 1.472}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.713, "count": 13, "max_ms": 50.007, "p95_ms": 50.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.19, "count": 13, "max_ms": 70.357, "p95_ms": 70.357}, "yolo11_640_fire_11class": {"avg_ms": 48.949, "count": 13, "max_ms": 65.824, "p95_ms": 65.824}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.316, "count": 13, "max_ms": 74.836, "p95_ms": 74.836}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.368, "count": 13, "max_ms": 101.438, "p95_ms": 101.438}, "read_interval": {"avg_ms": 2796.354, "count": 13, "max_ms": 18145.507, "p95_ms": 18145.507}, "triton_wait": {"avg_ms": 50.601, "count": 13, "max_ms": 74.848, "p95_ms": 74.848}, "visualize": {"avg_ms": 3.549, "count": 13, "max_ms": 5.018, "p95_ms": 5.018}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.306, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10541.961, "limit_mb": 8796093022207.996}, "detail": {"pid": 69735, "restart_count": 148, "start_count": 149}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 153, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:53:24+09:00", "last_error_summary": "[h264 @ 0x55d00beb5580] error while decoding MB 54 24, bytestream -11 [in#0/rtsp @ 0x55d00be73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d00be73e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:53:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:53:27+09:00", "pid": 69735, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.176, "name": "ffmpeg", "pid": 69735, "rss_mb": 206.176, "state": "S (sleeping)", "threads": 106}, "restart_count": 148, "restart_count_per_hour": 10, "rtsp_timeout_count": 240, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 149}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.386, "sample_count": 2303, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1335498, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:53:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.709, "last_frame_pushed_at": "2026-07-01T00:53:24+09:00", "last_frame_read_age_sec": 4.793, "last_frame_read_at": "2026-07-01T00:53:24+09:00", "mjpeg": {"age_sec": 4.696, "exists": true, "mtime": "2026-07-01T00:53:24.768970+09:00", "mtime_epoch": 1782834804.7689705, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93725}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:53:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1690.137, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.565, "pushed": 0.565, "read": 0.565}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.46, "count": 13, "max_ms": 23.616, "p95_ms": 23.616}, "gstreamer_push": {"avg_ms": 1.25, "count": 13, "max_ms": 1.472, "p95_ms": 1.472}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.713, "count": 13, "max_ms": 50.007, "p95_ms": 50.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.19, "count": 13, "max_ms": 70.357, "p95_ms": 70.357}, "yolo11_640_fire_11class": {"avg_ms": 48.949, "count": 13, "max_ms": 65.824, "p95_ms": 65.824}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.316, "count": 13, "max_ms": 74.836, "p95_ms": 74.836}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.368, "count": 13, "max_ms": 101.438, "p95_ms": 101.438}, "read_interval": {"avg_ms": 2796.354, "count": 13, "max_ms": 18145.507, "p95_ms": 18145.507}, "triton_wait": {"avg_ms": 50.601, "count": 13, "max_ms": 74.848, "p95_ms": 74.848}, "visualize": {"avg_ms": 3.549, "count": 13, "max_ms": 5.018, "p95_ms": 5.018}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.001, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10384.48, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55bb1e7fa600] error while decoding MB 109 16, bytestream -5 [in#0/rtsp @ 0x55bb1de66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb1de66e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 154, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:58:41+09:00", "last_error_summary": "[h264 @ 0x55bb1e7fa600] error while decoding MB 109 16, bytestream -5 [in#0/rtsp @ 0x55bb1de66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb1de66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:53:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 314.027, "last_started_at": "2026-07-01T00:53:27+09:00", "pid": 69735, "process": {"exists": false, "pid": 69735}, "restart_count": 148, "restart_count_per_hour": 9, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 149}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.997, "sample_count": 2558, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1338186, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:58:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-01T00:58:41+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T00:58:41+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T00:58:41.428751+09:00", "mtime_epoch": 1782835121.428751, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95927}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:58:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1691.426, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 59, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.972, "pushed": 1.972, "read": 1.972}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.448, "count": 59, "max_ms": 22.289, "p95_ms": 21.285}, "gstreamer_push": {"avg_ms": 1.267, "count": 59, "max_ms": 1.469, "p95_ms": 1.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.819, "count": 59, "max_ms": 68.281, "p95_ms": 65.074}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.237, "count": 59, "max_ms": 74.876, "p95_ms": 68.697}, "yolo11_640_fire_11class": {"avg_ms": 61.35, "count": 59, "max_ms": 73.577, "p95_ms": 67.774}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.181, "count": 59, "max_ms": 83.899, "p95_ms": 77.772}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.034, "count": 59, "max_ms": 109.953, "p95_ms": 106.542}, "read_interval": {"avg_ms": 507.844, "count": 59, "max_ms": 14621.071, "p95_ms": 124.879}, "triton_wait": {"avg_ms": 69.275, "count": 59, "max_ms": 83.912, "p95_ms": 77.784}, "visualize": {"avg_ms": 3.48, "count": 59, "max_ms": 3.785, "p95_ms": 3.622}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10384.301, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T00:58:41+09:00", "last_error_summary": "[h264 @ 0x55bb1e7fa600] error while decoding MB 109 16, bytestream -5 [in#0/rtsp @ 0x55bb1de66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb1de66e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 154, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:58:41+09:00", "last_error_summary": "[h264 @ 0x55bb1e7fa600] error while decoding MB 109 16, bytestream -5 [in#0/rtsp @ 0x55bb1de66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb1de66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:58:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 314.054, "last_started_at": "2026-07-01T00:53:27+09:00", "pid": 69735, "process": {"exists": false, "pid": 69735}, "restart_count": 148, "restart_count_per_hour": 9, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 149}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.997, "sample_count": 2559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1338187, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:58:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T00:58:41+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-01T00:58:41+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-01T00:58:41.428751+09:00", "mtime_epoch": 1782835121.428751, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95927}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:58:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1691.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.003, "pushed": 2.003, "read": 1.97}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.512, "count": 60, "max_ms": 23.236, "p95_ms": 21.285}, "gstreamer_push": {"avg_ms": 1.268, "count": 60, "max_ms": 1.469, "p95_ms": 1.363}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.716, "count": 60, "max_ms": 68.281, "p95_ms": 63.22}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.995, "count": 60, "max_ms": 74.876, "p95_ms": 68.639}, "yolo11_640_fire_11class": {"avg_ms": 61.122, "count": 60, "max_ms": 73.577, "p95_ms": 67.515}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.823, "count": 60, "max_ms": 83.899, "p95_ms": 77.584}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.78, "count": 60, "max_ms": 109.953, "p95_ms": 104.767}, "read_interval": {"avg_ms": 507.844, "count": 59, "max_ms": 14621.071, "p95_ms": 124.879}, "triton_wait": {"avg_ms": 68.915, "count": 60, "max_ms": 83.912, "p95_ms": 77.597}, "visualize": {"avg_ms": 3.522, "count": 60, "max_ms": 5.944, "p95_ms": 3.622}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10384.148, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 154, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:58:41+09:00", "last_error_summary": "[h264 @ 0x55bb1e7fa600] error while decoding MB 109 16, bytestream -5 [in#0/rtsp @ 0x55bb1de66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb1de66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:58:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 314.061, "last_started_at": "2026-07-01T00:53:27+09:00", "pid": 69735, "process": {"exists": false, "pid": 69735}, "restart_count": 148, "restart_count_per_hour": 9, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 149}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.997, "sample_count": 2559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1338187, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:58:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-01T00:58:41+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T00:58:41+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-01T00:58:41.428751+09:00", "mtime_epoch": 1782835121.428751, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95927}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:58:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1691.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.003, "pushed": 2.003, "read": 1.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.512, "count": 60, "max_ms": 23.236, "p95_ms": 21.285}, "gstreamer_push": {"avg_ms": 1.268, "count": 60, "max_ms": 1.469, "p95_ms": 1.363}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.716, "count": 60, "max_ms": 68.281, "p95_ms": 63.22}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.995, "count": 60, "max_ms": 74.876, "p95_ms": 68.639}, "yolo11_640_fire_11class": {"avg_ms": 61.122, "count": 60, "max_ms": 73.577, "p95_ms": 67.515}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.823, "count": 60, "max_ms": 83.899, "p95_ms": 77.584}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.78, "count": 60, "max_ms": 109.953, "p95_ms": 104.767}, "read_interval": {"avg_ms": 507.844, "count": 59, "max_ms": 14621.071, "p95_ms": 124.879}, "triton_wait": {"avg_ms": 68.915, "count": 60, "max_ms": 83.912, "p95_ms": 77.597}, "visualize": {"avg_ms": 3.522, "count": 60, "max_ms": 5.944, "p95_ms": 3.622}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10508.281, "limit_mb": 8796093022207.996}, "detail": {"pid": 69887, "restart_count": 149, "start_count": 150}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 154, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:58:41+09:00", "last_error_summary": "[h264 @ 0x55bb1e7fa600] error while decoding MB 109 16, bytestream -5 [in#0/rtsp @ 0x55bb1de66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb1de66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:58:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T00:58:43+09:00", "pid": 69887, "process": {"exists": true, "fd_count": 5, "hwm_mb": 148.863, "name": "ffmpeg", "pid": 69887, "rss_mb": 148.863, "state": "R (running)", "threads": 49}, "restart_count": 149, "restart_count_per_hour": 10, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.999, "sample_count": 2519, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1338187, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T00:58:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.476, "last_frame_pushed_at": "2026-07-01T00:58:41+09:00", "last_frame_read_age_sec": 4.559, "last_frame_read_at": "2026-07-01T00:58:41+09:00", "mjpeg": {"age_sec": 4.46, "exists": true, "mtime": "2026-07-01T00:58:41.524752+09:00", "mtime_epoch": 1782835121.5247521, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94827}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T00:58:45+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1691.426, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 22, "pushed_frames": 22, "read_frames": 21, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.735, "pushed": 0.735, "read": 0.702}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.345, "count": 22, "max_ms": 23.236, "p95_ms": 22.289}, "gstreamer_push": {"avg_ms": 1.262, "count": 22, "max_ms": 1.402, "p95_ms": 1.363}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.208, "count": 22, "max_ms": 68.281, "p95_ms": 55.422}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.684, "count": 22, "max_ms": 67.762, "p95_ms": 66.033}, "yolo11_640_fire_11class": {"avg_ms": 53.343, "count": 22, "max_ms": 67.774, "p95_ms": 65.466}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.884, "count": 22, "max_ms": 77.439, "p95_ms": 76.068}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.831, "count": 22, "max_ms": 106.69, "p95_ms": 103.967}, "read_interval": {"avg_ms": 1213.266, "count": 21, "max_ms": 14621.071, "p95_ms": 8948.619}, "triton_wait": {"avg_ms": 58.116, "count": 22, "max_ms": 77.449, "p95_ms": 76.08}, "visualize": {"avg_ms": 3.594, "count": 22, "max_ms": 5.944, "p95_ms": 3.644}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9963}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10417.477, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56197f9cb140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f9cb140] error while decoding MB 0 38, bytestream 3670 [h264 @ 0x56197f8cd0c0] error while decoding MB 62 26, bytestream -14 [h264 @ 0x5619805a4c40] top block unavailable for requested intra mode -1 [h264 @ 0x5619805a4c40] error while decoding MB 69 0, bytestream 7283 [h264 @ 0x56197f933f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f933f80] error while decoding MB 0 5, bytestr", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 352, "h264_decode_error_count": 1133, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:00:22+09:00", "last_error_summary": "[h264 @ 0x56197f9cb140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f9cb140] error while decoding MB 0 38, bytestream 3670 [h264 @ 0x56197f8cd0c0] error while decoding MB 62 26, bytestream -14 [h264 @ 0x5619805a4c40] top block unavailable for requested intra mode -1 [h264 @ 0x5619805a4c40] error while decoding MB 69 0, bytestream 7283 [h264 @ 0x56197f933f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f933f80] error while decoding MB 0 5, bytestr", "last_exit_at": "2026-07-01T00:47:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 782.827, "last_started_at": "2026-07-01T00:47:19+09:00", "pid": 69476, "process": {"exists": false, "pid": 69476}, "restart_count": 385, "restart_count_per_hour": 4, "rtsp_timeout_count": 322, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 386}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.953, "sample_count": 2578, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1195654, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:00:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-01T01:00:22+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-01T01:00:22+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-01T01:00:22.290137+09:00", "mtime_epoch": 1782835222.2901373, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:00:22+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4934.672, "name": "celery", "pid": 267, "rss_mb": 4934.672, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.203, "pushed": 3.203, "read": 3.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.884, "count": 96, "max_ms": 22.992, "p95_ms": 20.88}, "gstreamer_push": {"avg_ms": 1.204, "count": 96, "max_ms": 1.313, "p95_ms": 1.291}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.678, "count": 96, "max_ms": 42.017, "p95_ms": 40.772}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.764, "count": 96, "max_ms": 71.852, "p95_ms": 68.546}, "yolo11_640_fire_11class": {"avg_ms": 62.221, "count": 96, "max_ms": 71.103, "p95_ms": 67.407}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.187, "count": 96, "max_ms": 82.344, "p95_ms": 77.021}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.471, "count": 96, "max_ms": 114.414, "p95_ms": 105.233}, "read_interval": {"avg_ms": 312.56, "count": 96, "max_ms": 19098.555, "p95_ms": 121.757}, "triton_wait": {"avg_ms": 71.211, "count": 96, "max_ms": 82.356, "p95_ms": 77.034}, "visualize": {"avg_ms": 3.394, "count": 96, "max_ms": 4.879, "p95_ms": 3.626}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9963}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10417.16, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:00:22+09:00", "last_error_summary": "[h264 @ 0x56197f9cb140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f9cb140] error while decoding MB 0 38, bytestream 3670 [h264 @ 0x56197f8cd0c0] error while decoding MB 62 26, bytestream -14 [h264 @ 0x5619805a4c40] top block unavailable for requested intra mode -1 [h264 @ 0x5619805a4c40] error while decoding MB 69 0, bytestream 7283 [h264 @ 0x56197f933f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f933f80] error while decoding MB 0 5, bytestr", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 352, "h264_decode_error_count": 1133, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:00:22+09:00", "last_error_summary": "[h264 @ 0x56197f9cb140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f9cb140] error while decoding MB 0 38, bytestream 3670 [h264 @ 0x56197f8cd0c0] error while decoding MB 62 26, bytestream -14 [h264 @ 0x5619805a4c40] top block unavailable for requested intra mode -1 [h264 @ 0x5619805a4c40] error while decoding MB 69 0, bytestream 7283 [h264 @ 0x56197f933f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f933f80] error while decoding MB 0 5, bytestr", "last_exit_at": "2026-07-01T01:00:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 782.88, "last_started_at": "2026-07-01T00:47:19+09:00", "pid": 69476, "process": {"exists": false, "pid": 69476}, "restart_count": 385, "restart_count_per_hour": 4, "rtsp_timeout_count": 322, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 386}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.953, "sample_count": 2578, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1195655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:00:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T01:00:22+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-01T01:00:22+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-01T01:00:22.290137+09:00", "mtime_epoch": 1782835222.2901373, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:00:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4934.672, "name": "celery", "pid": 267, "rss_mb": 4934.672, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.21, "pushed": 3.21, "read": 3.177}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.875, "count": 96, "max_ms": 22.992, "p95_ms": 20.88}, "gstreamer_push": {"avg_ms": 1.204, "count": 96, "max_ms": 1.313, "p95_ms": 1.291}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.664, "count": 96, "max_ms": 42.017, "p95_ms": 40.772}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.856, "count": 96, "max_ms": 71.852, "p95_ms": 68.546}, "yolo11_640_fire_11class": {"avg_ms": 62.29, "count": 96, "max_ms": 71.103, "p95_ms": 67.407}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.268, "count": 96, "max_ms": 82.344, "p95_ms": 77.021}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.534, "count": 96, "max_ms": 114.414, "p95_ms": 105.233}, "read_interval": {"avg_ms": 314.687, "count": 95, "max_ms": 19098.555, "p95_ms": 121.757}, "triton_wait": {"avg_ms": 71.291, "count": 96, "max_ms": 82.356, "p95_ms": 77.034}, "visualize": {"avg_ms": 3.388, "count": 96, "max_ms": 4.879, "p95_ms": 3.562}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9963}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10417.293, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 352, "h264_decode_error_count": 1133, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:00:22+09:00", "last_error_summary": "[h264 @ 0x56197f9cb140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f9cb140] error while decoding MB 0 38, bytestream 3670 [h264 @ 0x56197f8cd0c0] error while decoding MB 62 26, bytestream -14 [h264 @ 0x5619805a4c40] top block unavailable for requested intra mode -1 [h264 @ 0x5619805a4c40] error while decoding MB 69 0, bytestream 7283 [h264 @ 0x56197f933f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f933f80] error while decoding MB 0 5, bytestr", "last_exit_at": "2026-07-01T01:00:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 782.887, "last_started_at": "2026-07-01T00:47:19+09:00", "pid": 69476, "process": {"exists": false, "pid": 69476}, "restart_count": 385, "restart_count_per_hour": 4, "rtsp_timeout_count": 322, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 386}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.953, "sample_count": 2578, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1195655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:00:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-01T01:00:22+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-01T01:00:22+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-01T01:00:22.290137+09:00", "mtime_epoch": 1782835222.2901373, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:00:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4934.672, "name": "celery", "pid": 267, "rss_mb": 4934.672, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.209, "pushed": 3.209, "read": 3.176}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.875, "count": 96, "max_ms": 22.992, "p95_ms": 20.88}, "gstreamer_push": {"avg_ms": 1.204, "count": 96, "max_ms": 1.313, "p95_ms": 1.291}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.664, "count": 96, "max_ms": 42.017, "p95_ms": 40.772}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.856, "count": 96, "max_ms": 71.852, "p95_ms": 68.546}, "yolo11_640_fire_11class": {"avg_ms": 62.29, "count": 96, "max_ms": 71.103, "p95_ms": 67.407}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.268, "count": 96, "max_ms": 82.344, "p95_ms": 77.021}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.534, "count": 96, "max_ms": 114.414, "p95_ms": 105.233}, "read_interval": {"avg_ms": 314.687, "count": 95, "max_ms": 19098.555, "p95_ms": 121.757}, "triton_wait": {"avg_ms": 71.291, "count": 96, "max_ms": 82.356, "p95_ms": 77.034}, "visualize": {"avg_ms": 3.388, "count": 96, "max_ms": 4.879, "p95_ms": 3.562}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9963}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10525.406, "limit_mb": 8796093022207.996}, "detail": {"pid": 70039, "restart_count": 386, "start_count": 387}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 352, "h264_decode_error_count": 1133, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:00:22+09:00", "last_error_summary": "[h264 @ 0x56197f9cb140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f9cb140] error while decoding MB 0 38, bytestream 3670 [h264 @ 0x56197f8cd0c0] error while decoding MB 62 26, bytestream -14 [h264 @ 0x5619805a4c40] top block unavailable for requested intra mode -1 [h264 @ 0x5619805a4c40] error while decoding MB 69 0, bytestream 7283 [h264 @ 0x56197f933f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56197f933f80] error while decoding MB 0 5, bytestr", "last_exit_at": "2026-07-01T01:00:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:00:25+09:00", "pid": 70039, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.074, "name": "ffmpeg", "pid": 70039, "rss_mb": 139.074, "state": "S (sleeping)", "threads": 106}, "restart_count": 386, "restart_count_per_hour": 5, "rtsp_timeout_count": 322, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 387}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.018, "sample_count": 2525, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1195655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:00:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.871, "last_frame_pushed_at": "2026-07-01T01:00:22+09:00", "last_frame_read_age_sec": 4.975, "last_frame_read_at": "2026-07-01T01:00:22+09:00", "mjpeg": {"age_sec": 4.857, "exists": true, "mtime": "2026-07-01T01:00:22.406139+09:00", "mtime_epoch": 1782835222.406139, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122709}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:00:27+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4934.672, "name": "celery", "pid": 267, "rss_mb": 4934.672, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 54, "pushed_frames": 54, "read_frames": 54, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.8, "pushed": 1.8, "read": 1.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.788, "count": 54, "max_ms": 22.992, "p95_ms": 21.567}, "gstreamer_push": {"avg_ms": 1.197, "count": 54, "max_ms": 1.313, "p95_ms": 1.283}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.829, "count": 54, "max_ms": 42.017, "p95_ms": 41.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.465, "count": 54, "max_ms": 71.852, "p95_ms": 68.84}, "yolo11_640_fire_11class": {"avg_ms": 61.832, "count": 54, "max_ms": 71.103, "p95_ms": 67.407}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.783, "count": 54, "max_ms": 82.344, "p95_ms": 77.164}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.135, "count": 54, "max_ms": 114.414, "p95_ms": 105.716}, "read_interval": {"avg_ms": 465.516, "count": 54, "max_ms": 19098.555, "p95_ms": 123.355}, "triton_wait": {"avg_ms": 70.815, "count": 54, "max_ms": 82.356, "p95_ms": 77.176}, "visualize": {"avg_ms": 3.365, "count": 54, "max_ms": 4.879, "p95_ms": 3.482}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10656.785, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 154, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:58:41+09:00", "last_error_summary": "[h264 @ 0x55bb1e7fa600] error while decoding MB 109 16, bytestream -5 [in#0/rtsp @ 0x55bb1de66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb1de66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:58:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 186.777, "last_started_at": "2026-07-01T00:58:43+09:00", "pid": 69887, "process": {"exists": true, "fd_count": 5, "hwm_mb": 328.07, "name": "ffmpeg", "pid": 69887, "rss_mb": 328.07, "state": "S (sleeping)", "threads": 106}, "restart_count": 149, "restart_count_per_hour": 10, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.648, "sample_count": 2161, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1339598, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:01:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.058, "last_frame_pushed_at": "2026-07-01T01:01:20+09:00", "last_frame_read_age_sec": 30.162, "last_frame_read_at": "2026-07-01T01:01:20+09:00", "mjpeg": {"age_sec": 30.038, "exists": true, "mtime": "2026-07-01T01:01:20.722970+09:00", "mtime_epoch": 1782835280.7229695, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86253}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:01:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1692.199, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10642.848, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 154, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T00:58:41+09:00", "last_error_summary": "[h264 @ 0x55bb1e7fa600] error while decoding MB 109 16, bytestream -5 [in#0/rtsp @ 0x55bb1de66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb1de66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:58:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 186.789, "last_started_at": "2026-07-01T00:58:43+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 149, "restart_count_per_hour": 10, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.648, "sample_count": 2161, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1339598, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:01:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.071, "last_frame_pushed_at": "2026-07-01T01:01:20+09:00", "last_frame_read_age_sec": 30.175, "last_frame_read_at": "2026-07-01T01:01:20+09:00", "mjpeg": {"age_sec": 30.051, "exists": true, "mtime": "2026-07-01T01:01:20.722970+09:00", "mtime_epoch": 1782835280.7229695, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86253}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:01:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1692.199, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10356.57, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561097bda7c0] error while decoding MB 112 29, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 150, "h264_decode_error_count": 155, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:01:50+09:00", "last_error_summary": "[h264 @ 0x561097bda7c0] error while decoding MB 112 29, bytestream -5", "last_exit_at": "2026-07-01T00:58:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 186.839, "last_started_at": "2026-07-01T00:58:43+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 149, "restart_count_per_hour": 10, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.648, "sample_count": 2161, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1339598, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:01:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.12, "last_frame_pushed_at": "2026-07-01T01:01:20+09:00", "last_frame_read_age_sec": 30.225, "last_frame_read_at": "2026-07-01T01:01:20+09:00", "mjpeg": {"age_sec": 30.1, "exists": true, "mtime": "2026-07-01T01:01:20.722970+09:00", "mtime_epoch": 1782835280.7229695, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86253}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:01:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1692.199, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11163}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10377.176, "limit_mb": 8796093022207.996}, "detail": {"pid": 70146, "restart_count": 150, "start_count": 151}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 150, "h264_decode_error_count": 155, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:01:50+09:00", "last_error_summary": "[h264 @ 0x561097bda7c0] error while decoding MB 112 29, bytestream -5", "last_exit_at": "2026-07-01T00:58:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:01:53+09:00", "pid": 70146, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.629, "name": "ffmpeg", "pid": 70146, "rss_mb": 57.629, "state": "S (sleeping)", "threads": 1}, "restart_count": 150, "restart_count_per_hour": 11, "rtsp_timeout_count": 242, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.695, "sample_count": 2120, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1339598, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:01:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 34.868, "last_frame_pushed_at": "2026-07-01T01:01:20+09:00", "last_frame_read_age_sec": 34.973, "last_frame_read_at": "2026-07-01T01:01:20+09:00", "mjpeg": {"age_sec": 34.848, "exists": true, "mtime": "2026-07-01T01:01:20.722970+09:00", "mtime_epoch": 1782835280.7229695, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86253}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:01:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1692.199, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10420.855, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b52c05c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c05c600] error while decoding MB 0 20, bytestream 4695 [h264 @ 0x55b52b7feb40] top block unavailable for requested intra mode [h264 @ 0x55b52b7feb40] error while decoding MB 44 0, bytestream 4670 [h264 @ 0x55b52c1fa240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c1fa240] error while decoding MB 0 10, bytestream 1188 [h264 @ 0x55b52b7d67c0] left block unavailable for requested in", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 353, "h264_decode_error_count": 1138, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:06:21+09:00", "last_error_summary": "[h264 @ 0x55b52c05c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c05c600] error while decoding MB 0 20, bytestream 4695 [h264 @ 0x55b52b7feb40] top block unavailable for requested intra mode [h264 @ 0x55b52b7feb40] error while decoding MB 44 0, bytestream 4670 [h264 @ 0x55b52c1fa240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c1fa240] error while decoding MB 0 10, bytestream 1188 [h264 @ 0x55b52b7d67c0] left block unavailable for requested in", "last_exit_at": "2026-07-01T01:00:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 356.344, "last_started_at": "2026-07-01T01:00:25+09:00", "pid": 70039, "process": {"exists": false, "pid": 70039}, "restart_count": 386, "restart_count_per_hour": 4, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 387}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.747, "sample_count": 2619, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1198743, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:06:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-01T01:06:21+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-01T01:06:21+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T01:06:21.543508+09:00", "mtime_epoch": 1782835581.5435076, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118004}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:06:21+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4936.219, "name": "celery", "pid": 267, "rss_mb": 4936.219, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 93, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.102, "pushed": 3.102, "read": 3.102}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.656, "count": 93, "max_ms": 26.878, "p95_ms": 25.441}, "gstreamer_push": {"avg_ms": 1.227, "count": 93, "max_ms": 1.36, "p95_ms": 1.316}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.865, "count": 93, "max_ms": 43.241, "p95_ms": 40.857}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.379, "count": 93, "max_ms": 68.474, "p95_ms": 66.699}, "yolo11_640_fire_11class": {"avg_ms": 54.547, "count": 93, "max_ms": 67.208, "p95_ms": 65.013}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.687, "count": 93, "max_ms": 77.43, "p95_ms": 75.01}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.397, "count": 93, "max_ms": 105.305, "p95_ms": 104.124}, "read_interval": {"avg_ms": 322.727, "count": 93, "max_ms": 20375.83, "p95_ms": 120.14}, "triton_wait": {"avg_ms": 60.698, "count": 93, "max_ms": 77.442, "p95_ms": 75.023}, "visualize": {"avg_ms": 3.369, "count": 93, "max_ms": 3.843, "p95_ms": 3.598}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10420.359, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:06:21+09:00", "last_error_summary": "[h264 @ 0x55b52c05c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c05c600] error while decoding MB 0 20, bytestream 4695 [h264 @ 0x55b52b7feb40] top block unavailable for requested intra mode [h264 @ 0x55b52b7feb40] error while decoding MB 44 0, bytestream 4670 [h264 @ 0x55b52c1fa240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c1fa240] error while decoding MB 0 10, bytestream 1188 [h264 @ 0x55b52b7d67c0] left block unavailable for requested in", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 353, "h264_decode_error_count": 1138, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:06:21+09:00", "last_error_summary": "[h264 @ 0x55b52c05c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c05c600] error while decoding MB 0 20, bytestream 4695 [h264 @ 0x55b52b7feb40] top block unavailable for requested intra mode [h264 @ 0x55b52b7feb40] error while decoding MB 44 0, bytestream 4670 [h264 @ 0x55b52c1fa240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c1fa240] error while decoding MB 0 10, bytestream 1188 [h264 @ 0x55b52b7d67c0] left block unavailable for requested in", "last_exit_at": "2026-07-01T01:06:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 356.372, "last_started_at": "2026-07-01T01:00:25+09:00", "pid": 70039, "process": {"exists": false, "pid": 70039}, "restart_count": 386, "restart_count_per_hour": 4, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 387}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.747, "sample_count": 2620, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1198744, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:06:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T01:06:21+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-01T01:06:21+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-01T01:06:21.543508+09:00", "mtime_epoch": 1782835581.5435076, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118004}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:06:21+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4936.219, "name": "celery", "pid": 267, "rss_mb": 4936.219, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 93, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.101, "pushed": 3.101, "read": 3.101}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.754, "count": 93, "max_ms": 29.514, "p95_ms": 25.562}, "gstreamer_push": {"avg_ms": 1.229, "count": 93, "max_ms": 1.429, "p95_ms": 1.319}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.82, "count": 93, "max_ms": 43.241, "p95_ms": 40.857}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.175, "count": 93, "max_ms": 68.474, "p95_ms": 66.699}, "yolo11_640_fire_11class": {"avg_ms": 54.341, "count": 93, "max_ms": 67.208, "p95_ms": 65.013}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.397, "count": 93, "max_ms": 77.43, "p95_ms": 75.01}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.254, "count": 93, "max_ms": 105.305, "p95_ms": 104.124}, "read_interval": {"avg_ms": 322.727, "count": 93, "max_ms": 20375.83, "p95_ms": 120.14}, "triton_wait": {"avg_ms": 60.409, "count": 93, "max_ms": 77.442, "p95_ms": 75.023}, "visualize": {"avg_ms": 3.416, "count": 93, "max_ms": 7.738, "p95_ms": 3.653}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10420.348, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 353, "h264_decode_error_count": 1138, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:06:21+09:00", "last_error_summary": "[h264 @ 0x55b52c05c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c05c600] error while decoding MB 0 20, bytestream 4695 [h264 @ 0x55b52b7feb40] top block unavailable for requested intra mode [h264 @ 0x55b52b7feb40] error while decoding MB 44 0, bytestream 4670 [h264 @ 0x55b52c1fa240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c1fa240] error while decoding MB 0 10, bytestream 1188 [h264 @ 0x55b52b7d67c0] left block unavailable for requested in", "last_exit_at": "2026-07-01T01:06:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 356.38, "last_started_at": "2026-07-01T01:00:25+09:00", "pid": 70039, "process": {"exists": false, "pid": 70039}, "restart_count": 386, "restart_count_per_hour": 4, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 387}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.747, "sample_count": 2620, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1198744, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:06:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T01:06:21+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-01T01:06:21+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-01T01:06:21.543508+09:00", "mtime_epoch": 1782835581.5435076, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118004}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:06:21+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4936.219, "name": "celery", "pid": 267, "rss_mb": 4936.219, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.11, "pushed": 3.11, "read": 3.077}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.754, "count": 93, "max_ms": 29.514, "p95_ms": 25.562}, "gstreamer_push": {"avg_ms": 1.229, "count": 93, "max_ms": 1.429, "p95_ms": 1.319}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.82, "count": 93, "max_ms": 43.241, "p95_ms": 40.857}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.175, "count": 93, "max_ms": 68.474, "p95_ms": 66.699}, "yolo11_640_fire_11class": {"avg_ms": 54.341, "count": 93, "max_ms": 67.208, "p95_ms": 65.013}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.397, "count": 93, "max_ms": 77.43, "p95_ms": 75.01}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.254, "count": 93, "max_ms": 105.305, "p95_ms": 104.124}, "read_interval": {"avg_ms": 324.967, "count": 92, "max_ms": 20375.83, "p95_ms": 120.14}, "triton_wait": {"avg_ms": 60.409, "count": 93, "max_ms": 77.442, "p95_ms": 75.023}, "visualize": {"avg_ms": 3.416, "count": 93, "max_ms": 7.738, "p95_ms": 3.653}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 9989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10528.305, "limit_mb": 8796093022207.996}, "detail": {"pid": 70298, "restart_count": 387, "start_count": 388}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 353, "h264_decode_error_count": 1138, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:06:21+09:00", "last_error_summary": "[h264 @ 0x55b52c05c600] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c05c600] error while decoding MB 0 20, bytestream 4695 [h264 @ 0x55b52b7feb40] top block unavailable for requested intra mode [h264 @ 0x55b52b7feb40] error while decoding MB 44 0, bytestream 4670 [h264 @ 0x55b52c1fa240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b52c1fa240] error while decoding MB 0 10, bytestream 1188 [h264 @ 0x55b52b7d67c0] left block unavailable for requested in", "last_exit_at": "2026-07-01T01:06:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:06:23+09:00", "pid": 70298, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.941, "name": "ffmpeg", "pid": 70298, "rss_mb": 138.941, "state": "S (sleeping)", "threads": 106}, "restart_count": 387, "restart_count_per_hour": 5, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 388}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.764, "sample_count": 2571, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1198744, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:06:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.31, "last_frame_pushed_at": "2026-07-01T01:06:21+09:00", "last_frame_read_age_sec": 4.4, "last_frame_read_at": "2026-07-01T01:06:21+09:00", "mjpeg": {"age_sec": 4.293, "exists": true, "mtime": "2026-07-01T01:06:21.643509+09:00", "mtime_epoch": 1782835581.6435091, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120143}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:06:25+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4936.219, "name": "celery", "pid": 267, "rss_mb": 4936.219, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 56, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.867, "pushed": 1.867, "read": 1.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.777, "count": 56, "max_ms": 29.514, "p95_ms": 26.165}, "gstreamer_push": {"avg_ms": 1.223, "count": 56, "max_ms": 1.429, "p95_ms": 1.319}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.301, "count": 56, "max_ms": 40.857, "p95_ms": 39.824}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.683, "count": 56, "max_ms": 67.177, "p95_ms": 64.789}, "yolo11_640_fire_11class": {"avg_ms": 48.994, "count": 56, "max_ms": 66.649, "p95_ms": 63.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.589, "count": 56, "max_ms": 77.43, "p95_ms": 73.608}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.32, "count": 56, "max_ms": 105.305, "p95_ms": 101.936}, "read_interval": {"avg_ms": 459.183, "count": 56, "max_ms": 20375.83, "p95_ms": 116.327}, "triton_wait": {"avg_ms": 52.6, "count": 56, "max_ms": 77.442, "p95_ms": 73.621}, "visualize": {"avg_ms": 3.41, "count": 56, "max_ms": 7.738, "p95_ms": 3.598}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11185}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10362.102, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x560ca8ec67c0] error while decoding MB 76 48, bytestream -5 [in#0/rtsp @ 0x560ca87a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ca87a2e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 151, "h264_decode_error_count": 156, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:07:08+09:00", "last_error_summary": "[h264 @ 0x560ca8ec67c0] error while decoding MB 76 48, bytestream -5 [in#0/rtsp @ 0x560ca87a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ca87a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T00:58:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 314.668, "last_started_at": "2026-07-01T01:01:53+09:00", "pid": 70146, "process": {"exists": false, "pid": 70146}, "restart_count": 150, "restart_count_per_hour": 10, "rtsp_timeout_count": 244, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.9, "sample_count": 2543, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1342246, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:07:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T01:07:08+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-01T01:07:08+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T01:07:08.176240+09:00", "mtime_epoch": 1782835628.17624, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91067}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:07:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1693.488, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 38, "pushed_frames": 38, "read_frames": 38, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.269, "pushed": 1.269, "read": 1.269}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.556, "count": 38, "max_ms": 23.126, "p95_ms": 19.463}, "gstreamer_push": {"avg_ms": 1.244, "count": 38, "max_ms": 1.381, "p95_ms": 1.343}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.855, "count": 38, "max_ms": 60.121, "p95_ms": 59.206}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.173, "count": 38, "max_ms": 68.045, "p95_ms": 67.425}, "yolo11_640_fire_11class": {"avg_ms": 62.409, "count": 38, "max_ms": 69.055, "p95_ms": 66.99}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.855, "count": 38, "max_ms": 77.429, "p95_ms": 77.0}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.546, "count": 38, "max_ms": 103.822, "p95_ms": 103.652}, "read_interval": {"avg_ms": 788.952, "count": 38, "max_ms": 15952.958, "p95_ms": 9966.505}, "triton_wait": {"avg_ms": 70.946, "count": 38, "max_ms": 77.442, "p95_ms": 77.012}, "visualize": {"avg_ms": 3.426, "count": 38, "max_ms": 3.665, "p95_ms": 3.624}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11185}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10361.801, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:07:08+09:00", "last_error_summary": "[h264 @ 0x560ca8ec67c0] error while decoding MB 76 48, bytestream -5 [in#0/rtsp @ 0x560ca87a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ca87a2e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 151, "h264_decode_error_count": 156, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:07:08+09:00", "last_error_summary": "[h264 @ 0x560ca8ec67c0] error while decoding MB 76 48, bytestream -5 [in#0/rtsp @ 0x560ca87a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ca87a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:07:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 314.714, "last_started_at": "2026-07-01T01:01:53+09:00", "pid": 70146, "process": {"exists": false, "pid": 70146}, "restart_count": 150, "restart_count_per_hour": 10, "rtsp_timeout_count": 244, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.9, "sample_count": 2544, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1342247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:07:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T01:07:08+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-01T01:07:08+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-01T01:07:08.176240+09:00", "mtime_epoch": 1782835628.17624, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91067}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:07:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1693.488, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 38, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.301, "pushed": 1.301, "read": 1.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.539, "count": 39, "max_ms": 23.126, "p95_ms": 19.463}, "gstreamer_push": {"avg_ms": 1.243, "count": 39, "max_ms": 1.381, "p95_ms": 1.343}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.971, "count": 39, "max_ms": 60.121, "p95_ms": 59.206}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.288, "count": 39, "max_ms": 68.045, "p95_ms": 67.665}, "yolo11_640_fire_11class": {"avg_ms": 62.526, "count": 39, "max_ms": 69.055, "p95_ms": 66.99}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.965, "count": 39, "max_ms": 77.429, "p95_ms": 77.0}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.637, "count": 39, "max_ms": 103.822, "p95_ms": 103.652}, "read_interval": {"avg_ms": 788.952, "count": 38, "max_ms": 15952.958, "p95_ms": 9966.505}, "triton_wait": {"avg_ms": 71.054, "count": 39, "max_ms": 77.442, "p95_ms": 77.012}, "visualize": {"avg_ms": 3.431, "count": 39, "max_ms": 3.665, "p95_ms": 3.624}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11185}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10361.762, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 151, "h264_decode_error_count": 156, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:07:08+09:00", "last_error_summary": "[h264 @ 0x560ca8ec67c0] error while decoding MB 76 48, bytestream -5 [in#0/rtsp @ 0x560ca87a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ca87a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:07:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 314.721, "last_started_at": "2026-07-01T01:01:53+09:00", "pid": 70146, "process": {"exists": false, "pid": 70146}, "restart_count": 150, "restart_count_per_hour": 10, "rtsp_timeout_count": 244, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.9, "sample_count": 2544, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1342247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:07:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T01:07:08+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-01T01:07:08+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-01T01:07:08.176240+09:00", "mtime_epoch": 1782835628.17624, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91067}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:07:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1693.488, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 38, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.3, "pushed": 1.3, "read": 1.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.539, "count": 39, "max_ms": 23.126, "p95_ms": 19.463}, "gstreamer_push": {"avg_ms": 1.243, "count": 39, "max_ms": 1.381, "p95_ms": 1.343}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.971, "count": 39, "max_ms": 60.121, "p95_ms": 59.206}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.288, "count": 39, "max_ms": 68.045, "p95_ms": 67.665}, "yolo11_640_fire_11class": {"avg_ms": 62.526, "count": 39, "max_ms": 69.055, "p95_ms": 66.99}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.965, "count": 39, "max_ms": 77.429, "p95_ms": 77.0}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.637, "count": 39, "max_ms": 103.822, "p95_ms": 103.652}, "read_interval": {"avg_ms": 788.952, "count": 38, "max_ms": 15952.958, "p95_ms": 9966.505}, "triton_wait": {"avg_ms": 71.054, "count": 39, "max_ms": 77.442, "p95_ms": 77.012}, "visualize": {"avg_ms": 3.431, "count": 39, "max_ms": 3.665, "p95_ms": 3.624}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11185}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10382.848, "limit_mb": 8796093022207.996}, "detail": {"pid": 70405, "restart_count": 151, "start_count": 152}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 151, "h264_decode_error_count": 156, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:07:08+09:00", "last_error_summary": "[h264 @ 0x560ca8ec67c0] error while decoding MB 76 48, bytestream -5 [in#0/rtsp @ 0x560ca87a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ca87a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:07:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:07:10+09:00", "pid": 70405, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.027, "name": "ffmpeg", "pid": 70405, "rss_mb": 58.027, "state": "S (sleeping)", "threads": 1}, "restart_count": 151, "restart_count_per_hour": 11, "rtsp_timeout_count": 244, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.893, "sample_count": 2506, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1342247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:07:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.393, "last_frame_pushed_at": "2026-07-01T01:07:08+09:00", "last_frame_read_age_sec": 4.496, "last_frame_read_at": "2026-07-01T01:07:08+09:00", "mjpeg": {"age_sec": 4.38, "exists": true, "mtime": "2026-07-01T01:07:08.292242+09:00", "mtime_epoch": 1782835628.2922418, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91508}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:07:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1693.488, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.602, "pushed": 0.602, "read": 0.602}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.742, "count": 13, "max_ms": 23.126, "p95_ms": 23.126}, "gstreamer_push": {"avg_ms": 1.233, "count": 13, "max_ms": 1.381, "p95_ms": 1.381}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.501, "count": 13, "max_ms": 55.079, "p95_ms": 55.079}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.608, "count": 13, "max_ms": 67.665, "p95_ms": 67.665}, "yolo11_640_fire_11class": {"avg_ms": 57.24, "count": 13, "max_ms": 66.966, "p95_ms": 66.966}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.649, "count": 13, "max_ms": 75.153, "p95_ms": 75.153}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.598, "count": 13, "max_ms": 101.078, "p95_ms": 101.078}, "read_interval": {"avg_ms": 2081.21, "count": 13, "max_ms": 15952.958, "p95_ms": 15952.958}, "triton_wait": {"avg_ms": 62.891, "count": 13, "max_ms": 75.166, "p95_ms": 75.166}, "visualize": {"avg_ms": 3.378, "count": 13, "max_ms": 3.605, "p95_ms": 3.605}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.585, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10032}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10425.668, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56355136a1c0] reference picture missing during reorder [h264 @ 0x56355136a1c0] Missing reference picture, default is 65565 [h264 @ 0x56355137e380] mmco: unref short failure [h264 @ 0x56355137e380] illegal short term buffer state detected [h264 @ 0x563551341fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563551341fc0] error while decoding MB 0 9, bytestream 39812 [h264 @ 0x563551bcc180] cabac decode of qscale diff failed at 65 31 [h264 @ 0x563551bcc180] error while ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 354, "h264_decode_error_count": 1144, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:16:05+09:00", "last_error_summary": "[h264 @ 0x56355136a1c0] reference picture missing during reorder [h264 @ 0x56355136a1c0] Missing reference picture, default is 65565 [h264 @ 0x56355137e380] mmco: unref short failure [h264 @ 0x56355137e380] illegal short term buffer state detected [h264 @ 0x563551341fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563551341fc0] error while decoding MB 0 9, bytestream 39812 [h264 @ 0x563551bcc180] cabac decode of qscale diff failed at 65 31 [h264 @ 0x563551bcc180] error while ", "last_exit_at": "2026-07-01T01:06:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 581.73, "last_started_at": "2026-07-01T01:06:23+09:00", "pid": 70298, "process": {"exists": false, "pid": 70298}, "restart_count": 387, "restart_count_per_hour": 5, "rtsp_timeout_count": 326, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 388}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.551, "sample_count": 2435, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1203899, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:16:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.084, "last_frame_pushed_at": "2026-07-01T01:16:05+09:00", "last_frame_read_age_sec": 0.074, "last_frame_read_at": "2026-07-01T01:16:05+09:00", "mjpeg": {"age_sec": 0.069, "exists": true, "mtime": "2026-07-01T01:16:05.597115+09:00", "mtime_epoch": 1782836165.5971148, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 139080}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:16:05+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4938.539, "name": "celery", "pid": 267, "rss_mb": 4938.539, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.7, "pushed": 3.7, "read": 3.7}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.062, "count": 110, "max_ms": 24.672, "p95_ms": 23.115}, "gstreamer_push": {"avg_ms": 1.231, "count": 111, "max_ms": 1.484, "p95_ms": 1.345}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.199, "count": 110, "max_ms": 42.652, "p95_ms": 41.402}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.306, "count": 110, "max_ms": 69.634, "p95_ms": 67.886}, "yolo11_640_fire_11class": {"avg_ms": 55.305, "count": 110, "max_ms": 68.66, "p95_ms": 66.755}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.575, "count": 110, "max_ms": 78.581, "p95_ms": 77.091}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.346, "count": 111, "max_ms": 145.239, "p95_ms": 105.397}, "read_interval": {"avg_ms": 270.479, "count": 111, "max_ms": 9228.57, "p95_ms": 122.534}, "triton_wait": {"avg_ms": 61.788, "count": 110, "max_ms": 78.594, "p95_ms": 77.105}, "visualize": {"avg_ms": 3.395, "count": 110, "max_ms": 5.364, "p95_ms": 3.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10032}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10425.496, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:16:05+09:00", "last_error_summary": "[h264 @ 0x56355136a1c0] reference picture missing during reorder [h264 @ 0x56355136a1c0] Missing reference picture, default is 65565 [h264 @ 0x56355137e380] mmco: unref short failure [h264 @ 0x56355137e380] illegal short term buffer state detected [h264 @ 0x563551341fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563551341fc0] error while decoding MB 0 9, bytestream 39812 [h264 @ 0x563551bcc180] cabac decode of qscale diff failed at 65 31 [h264 @ 0x563551bcc180] error while ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 354, "h264_decode_error_count": 1144, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:16:05+09:00", "last_error_summary": "[h264 @ 0x56355136a1c0] reference picture missing during reorder [h264 @ 0x56355136a1c0] Missing reference picture, default is 65565 [h264 @ 0x56355137e380] mmco: unref short failure [h264 @ 0x56355137e380] illegal short term buffer state detected [h264 @ 0x563551341fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563551341fc0] error while decoding MB 0 9, bytestream 39812 [h264 @ 0x563551bcc180] cabac decode of qscale diff failed at 65 31 [h264 @ 0x563551bcc180] error while ", "last_exit_at": "2026-07-01T01:16:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 581.756, "last_started_at": "2026-07-01T01:06:23+09:00", "pid": 70298, "process": {"exists": false, "pid": 70298}, "restart_count": 387, "restart_count_per_hour": 5, "rtsp_timeout_count": 326, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 388}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.551, "sample_count": 2436, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1203900, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:16:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T01:16:05+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T01:16:05+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-01T01:16:05.597115+09:00", "mtime_epoch": 1782836165.5971148, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 139080}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:16:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4938.539, "name": "celery", "pid": 267, "rss_mb": 4938.539, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.711, "pushed": 3.711, "read": 3.678}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.112, "count": 111, "max_ms": 26.579, "p95_ms": 23.253}, "gstreamer_push": {"avg_ms": 1.234, "count": 111, "max_ms": 1.489, "p95_ms": 1.345}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.187, "count": 111, "max_ms": 42.652, "p95_ms": 41.402}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.27, "count": 111, "max_ms": 69.634, "p95_ms": 67.886}, "yolo11_640_fire_11class": {"avg_ms": 55.27, "count": 111, "max_ms": 68.66, "p95_ms": 66.755}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.498, "count": 111, "max_ms": 78.581, "p95_ms": 77.091}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.263, "count": 111, "max_ms": 145.239, "p95_ms": 105.397}, "read_interval": {"avg_ms": 271.898, "count": 110, "max_ms": 9228.57, "p95_ms": 122.534}, "triton_wait": {"avg_ms": 61.708, "count": 111, "max_ms": 78.594, "p95_ms": 77.105}, "visualize": {"avg_ms": 3.422, "count": 111, "max_ms": 6.404, "p95_ms": 3.683}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10032}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10425.605, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 354, "h264_decode_error_count": 1144, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:16:05+09:00", "last_error_summary": "[h264 @ 0x56355136a1c0] reference picture missing during reorder [h264 @ 0x56355136a1c0] Missing reference picture, default is 65565 [h264 @ 0x56355137e380] mmco: unref short failure [h264 @ 0x56355137e380] illegal short term buffer state detected [h264 @ 0x563551341fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563551341fc0] error while decoding MB 0 9, bytestream 39812 [h264 @ 0x563551bcc180] cabac decode of qscale diff failed at 65 31 [h264 @ 0x563551bcc180] error while ", "last_exit_at": "2026-07-01T01:16:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 581.765, "last_started_at": "2026-07-01T01:06:23+09:00", "pid": 70298, "process": {"exists": false, "pid": 70298}, "restart_count": 387, "restart_count_per_hour": 5, "rtsp_timeout_count": 326, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 388}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.551, "sample_count": 2436, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1203900, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:16:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T01:16:05+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-01T01:16:05+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-01T01:16:05.597115+09:00", "mtime_epoch": 1782836165.5971148, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 139080}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:16:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4938.539, "name": "celery", "pid": 267, "rss_mb": 4938.539, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.71, "pushed": 3.71, "read": 3.677}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.112, "count": 111, "max_ms": 26.579, "p95_ms": 23.253}, "gstreamer_push": {"avg_ms": 1.234, "count": 111, "max_ms": 1.489, "p95_ms": 1.345}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.187, "count": 111, "max_ms": 42.652, "p95_ms": 41.402}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.27, "count": 111, "max_ms": 69.634, "p95_ms": 67.886}, "yolo11_640_fire_11class": {"avg_ms": 55.27, "count": 111, "max_ms": 68.66, "p95_ms": 66.755}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.498, "count": 111, "max_ms": 78.581, "p95_ms": 77.091}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.263, "count": 111, "max_ms": 145.239, "p95_ms": 105.397}, "read_interval": {"avg_ms": 271.898, "count": 110, "max_ms": 9228.57, "p95_ms": 122.534}, "triton_wait": {"avg_ms": 61.708, "count": 111, "max_ms": 78.594, "p95_ms": 77.105}, "visualize": {"avg_ms": 3.422, "count": 111, "max_ms": 6.404, "p95_ms": 3.683}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10032}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10534.387, "limit_mb": 8796093022207.996}, "detail": {"pid": 70557, "restart_count": 388, "start_count": 389}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 354, "h264_decode_error_count": 1144, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:16:05+09:00", "last_error_summary": "[h264 @ 0x56355136a1c0] reference picture missing during reorder [h264 @ 0x56355136a1c0] Missing reference picture, default is 65565 [h264 @ 0x56355137e380] mmco: unref short failure [h264 @ 0x56355137e380] illegal short term buffer state detected [h264 @ 0x563551341fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563551341fc0] error while decoding MB 0 9, bytestream 39812 [h264 @ 0x563551bcc180] cabac decode of qscale diff failed at 65 31 [h264 @ 0x563551bcc180] error while ", "last_exit_at": "2026-07-01T01:16:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:16:08+09:00", "pid": 70557, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.832, "name": "ffmpeg", "pid": 70557, "rss_mb": 138.832, "state": "S (sleeping)", "threads": 106}, "restart_count": 388, "restart_count_per_hour": 6, "rtsp_timeout_count": 326, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 389}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.552, "sample_count": 2393, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1203900, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:16:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.96, "last_frame_pushed_at": "2026-07-01T01:16:05+09:00", "last_frame_read_age_sec": 5.053, "last_frame_read_at": "2026-07-01T01:16:05+09:00", "mjpeg": {"age_sec": 4.944, "exists": true, "mtime": "2026-07-01T01:16:05.701117+09:00", "mtime_epoch": 1782836165.7011173, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 140110}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:16:10+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4938.539, "name": "celery", "pid": 267, "rss_mb": 4938.539, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 69, "pushed_frames": 69, "read_frames": 68, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.303, "pushed": 2.303, "read": 2.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.811, "count": 69, "max_ms": 26.579, "p95_ms": 23.687}, "gstreamer_push": {"avg_ms": 1.242, "count": 69, "max_ms": 1.489, "p95_ms": 1.383}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.972, "count": 69, "max_ms": 40.483, "p95_ms": 32.94}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.046, "count": 69, "max_ms": 68.235, "p95_ms": 65.816}, "yolo11_640_fire_11class": {"avg_ms": 50.03, "count": 69, "max_ms": 67.714, "p95_ms": 64.931}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.089, "count": 69, "max_ms": 78.581, "p95_ms": 77.091}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.854, "count": 69, "max_ms": 145.239, "p95_ms": 105.242}, "read_interval": {"avg_ms": 367.718, "count": 68, "max_ms": 9228.57, "p95_ms": 125.043}, "triton_wait": {"avg_ms": 54.421, "count": 69, "max_ms": 78.594, "p95_ms": 77.105}, "visualize": {"avg_ms": 3.451, "count": 69, "max_ms": 6.404, "p95_ms": 3.972}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11228}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10368.562, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x5625252d3200] missing picture in access unit with size 18 [h264 @ 0x562525462580] no frame! [in#0/rtsp @ 0x5625252cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5625252cee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5625252d3840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 152, "h264_decode_error_count": 157, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:17:10+09:00", "last_error_summary": "[NULL @ 0x5625252d3200] missing picture in access unit with size 18 [h264 @ 0x562525462580] no frame! [in#0/rtsp @ 0x5625252cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5625252cee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5625252d3840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T01:07:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 599.424, "last_started_at": "2026-07-01T01:07:10+09:00", "pid": 70405, "process": {"exists": false, "pid": 70405}, "restart_count": 151, "restart_count_per_hour": 9, "rtsp_timeout_count": 246, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.478, "sample_count": 2419, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1347465, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:17:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T01:17:10+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T01:17:10+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-01T01:17:10.034524+09:00", "mtime_epoch": 1782836230.0345237, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:17:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1696.066, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 28, "pushed_frames": 28, "read_frames": 28, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.936, "pushed": 0.936, "read": 0.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.374, "count": 28, "max_ms": 21.532, "p95_ms": 21.424}, "gstreamer_push": {"avg_ms": 1.245, "count": 28, "max_ms": 1.332, "p95_ms": 1.32}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.431, "count": 28, "max_ms": 57.734, "p95_ms": 56.991}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.528, "count": 28, "max_ms": 69.0, "p95_ms": 68.037}, "yolo11_640_fire_11class": {"avg_ms": 57.975, "count": 28, "max_ms": 66.094, "p95_ms": 66.086}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.876, "count": 28, "max_ms": 76.792, "p95_ms": 76.466}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.319, "count": 28, "max_ms": 103.363, "p95_ms": 103.161}, "read_interval": {"avg_ms": 1069.983, "count": 28, "max_ms": 27089.396, "p95_ms": 118.803}, "triton_wait": {"avg_ms": 64.887, "count": 28, "max_ms": 76.805, "p95_ms": 76.479}, "visualize": {"avg_ms": 3.426, "count": 28, "max_ms": 3.584, "p95_ms": 3.584}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11228}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10368.234, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:17:10+09:00", "last_error_summary": "[NULL @ 0x5625252d3200] missing picture in access unit with size 18 [h264 @ 0x562525462580] no frame! [in#0/rtsp @ 0x5625252cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5625252cee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5625252d3840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 152, "h264_decode_error_count": 157, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:17:10+09:00", "last_error_summary": "[NULL @ 0x5625252d3200] missing picture in access unit with size 18 [h264 @ 0x562525462580] no frame! [in#0/rtsp @ 0x5625252cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5625252cee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5625252d3840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T01:17:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 599.452, "last_started_at": "2026-07-01T01:07:10+09:00", "pid": 70405, "process": {"exists": false, "pid": 70405}, "restart_count": 151, "restart_count_per_hour": 9, "rtsp_timeout_count": 246, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.478, "sample_count": 2419, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1347466, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:17:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T01:17:10+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T01:17:10+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T01:17:10.034524+09:00", "mtime_epoch": 1782836230.0345237, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:17:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1696.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 29, "pushed_frames": 29, "read_frames": 28, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.968, "pushed": 0.968, "read": 0.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.533, "count": 29, "max_ms": 23.992, "p95_ms": 21.532}, "gstreamer_push": {"avg_ms": 1.25, "count": 29, "max_ms": 1.398, "p95_ms": 1.332}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.125, "count": 29, "max_ms": 57.734, "p95_ms": 56.991}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.085, "count": 29, "max_ms": 69.0, "p95_ms": 68.037}, "yolo11_640_fire_11class": {"avg_ms": 57.567, "count": 29, "max_ms": 66.094, "p95_ms": 66.086}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.278, "count": 29, "max_ms": 76.792, "p95_ms": 76.466}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.996, "count": 29, "max_ms": 103.363, "p95_ms": 103.161}, "read_interval": {"avg_ms": 1069.983, "count": 28, "max_ms": 27089.396, "p95_ms": 118.803}, "triton_wait": {"avg_ms": 64.29, "count": 29, "max_ms": 76.805, "p95_ms": 76.479}, "visualize": {"avg_ms": 3.519, "count": 29, "max_ms": 6.128, "p95_ms": 3.584}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11228}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10368.238, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 152, "h264_decode_error_count": 157, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:17:10+09:00", "last_error_summary": "[NULL @ 0x5625252d3200] missing picture in access unit with size 18 [h264 @ 0x562525462580] no frame! [in#0/rtsp @ 0x5625252cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5625252cee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5625252d3840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T01:17:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 599.459, "last_started_at": "2026-07-01T01:07:10+09:00", "pid": 70405, "process": {"exists": false, "pid": 70405}, "restart_count": 151, "restart_count_per_hour": 9, "rtsp_timeout_count": 246, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.478, "sample_count": 2419, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1347466, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:17:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-01T01:17:10+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-01T01:17:10+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-01T01:17:10.034524+09:00", "mtime_epoch": 1782836230.0345237, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:17:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1696.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 29, "pushed_frames": 29, "read_frames": 28, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.968, "pushed": 0.968, "read": 0.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.533, "count": 29, "max_ms": 23.992, "p95_ms": 21.532}, "gstreamer_push": {"avg_ms": 1.25, "count": 29, "max_ms": 1.398, "p95_ms": 1.332}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.125, "count": 29, "max_ms": 57.734, "p95_ms": 56.991}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.085, "count": 29, "max_ms": 69.0, "p95_ms": 68.037}, "yolo11_640_fire_11class": {"avg_ms": 57.567, "count": 29, "max_ms": 66.094, "p95_ms": 66.086}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.278, "count": 29, "max_ms": 76.792, "p95_ms": 76.466}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.996, "count": 29, "max_ms": 103.363, "p95_ms": 103.161}, "read_interval": {"avg_ms": 1069.983, "count": 28, "max_ms": 27089.396, "p95_ms": 118.803}, "triton_wait": {"avg_ms": 64.29, "count": 29, "max_ms": 76.805, "p95_ms": 76.479}, "visualize": {"avg_ms": 3.519, "count": 29, "max_ms": 6.128, "p95_ms": 3.584}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11228}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10548.031, "limit_mb": 8796093022207.996}, "detail": {"pid": 70664, "restart_count": 152, "start_count": 153}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 152, "h264_decode_error_count": 157, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:17:10+09:00", "last_error_summary": "[NULL @ 0x5625252d3200] missing picture in access unit with size 18 [h264 @ 0x562525462580] no frame! [in#0/rtsp @ 0x5625252cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5625252cee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5625252d3840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T01:17:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:17:12+09:00", "pid": 70664, "process": {"exists": true, "fd_count": 5, "hwm_mb": 208.953, "name": "ffmpeg", "pid": 70664, "rss_mb": 208.953, "state": "S (sleeping)", "threads": 106}, "restart_count": 152, "restart_count_per_hour": 10, "rtsp_timeout_count": 246, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 153}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.483, "sample_count": 2381, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1347466, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:17:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.405, "last_frame_pushed_at": "2026-07-01T01:17:10+09:00", "last_frame_read_age_sec": 4.491, "last_frame_read_at": "2026-07-01T01:17:10+09:00", "mjpeg": {"age_sec": 4.389, "exists": true, "mtime": "2026-07-01T01:17:10.130526+09:00", "mtime_epoch": 1782836230.1305258, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90922}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:17:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1696.066, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.182, "pushed": 2.182, "read": 2.182}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.095, "count": 12, "max_ms": 23.992, "p95_ms": 23.992}, "gstreamer_push": {"avg_ms": 1.276, "count": 12, "max_ms": 1.398, "p95_ms": 1.398}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.567, "count": 12, "max_ms": 47.716, "p95_ms": 47.716}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.366, "count": 12, "max_ms": 54.816, "p95_ms": 54.816}, "yolo11_640_fire_11class": {"avg_ms": 47.779, "count": 12, "max_ms": 54.494, "p95_ms": 54.494}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.014, "count": 12, "max_ms": 55.415, "p95_ms": 55.415}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.485, "count": 12, "max_ms": 84.587, "p95_ms": 84.587}, "read_interval": {"avg_ms": 2341.402, "count": 12, "max_ms": 27089.396, "p95_ms": 27089.396}, "triton_wait": {"avg_ms": 49.024, "count": 12, "max_ms": 55.424, "p95_ms": 55.424}, "visualize": {"avg_ms": 3.644, "count": 12, "max_ms": 6.128, "p95_ms": 6.128}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.498, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10050}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10428.316, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55dc1e599280] top block unavailable for requested intra mode -1 [h264 @ 0x55dc1e599280] error while decoding MB 58 0, bytestream 8703 [h264 @ 0x55dc1eec9400] left block unavailable for requested intra mode [h264 @ 0x55dc1eec9400] error while decoding MB 0 9, bytestream 26724 [h264 @ 0x55dc1e672ac0] error while decoding MB 9 2, bytestream -10 [in#0/rtsp @ 0x55dc1e513e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dc1e513e00] Error retrieving a packet from demuxer: Conn", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 355, "h264_decode_error_count": 1147, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:20:30+09:00", "last_error_summary": "[h264 @ 0x55dc1e599280] top block unavailable for requested intra mode -1 [h264 @ 0x55dc1e599280] error while decoding MB 58 0, bytestream 8703 [h264 @ 0x55dc1eec9400] left block unavailable for requested intra mode [h264 @ 0x55dc1eec9400] error while decoding MB 0 9, bytestream 26724 [h264 @ 0x55dc1e672ac0] error while decoding MB 9 2, bytestream -10 [in#0/rtsp @ 0x55dc1e513e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dc1e513e00] Error retrieving a packet from demuxer: Conn", "last_exit_at": "2026-07-01T01:16:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 261.984, "last_started_at": "2026-07-01T01:16:08+09:00", "pid": 70557, "process": {"exists": false, "pid": 70557}, "restart_count": 388, "restart_count_per_hour": 5, "rtsp_timeout_count": 328, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 389}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.309, "sample_count": 2335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1206080, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:20:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T01:20:30+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T01:20:30+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T01:20:30.566734+09:00", "mtime_epoch": 1782836430.566734, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121355}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:20:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4939.57, "name": "celery", "pid": 267, "rss_mb": 4939.57, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.27, "pushed": 3.27, "read": 3.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.365, "count": 98, "max_ms": 25.591, "p95_ms": 25.104}, "gstreamer_push": {"avg_ms": 1.219, "count": 98, "max_ms": 1.342, "p95_ms": 1.309}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.645, "count": 98, "max_ms": 44.679, "p95_ms": 41.505}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.565, "count": 98, "max_ms": 68.969, "p95_ms": 67.974}, "yolo11_640_fire_11class": {"avg_ms": 61.221, "count": 98, "max_ms": 70.018, "p95_ms": 66.828}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.035, "count": 98, "max_ms": 77.964, "p95_ms": 76.715}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.511, "count": 98, "max_ms": 106.838, "p95_ms": 105.016}, "read_interval": {"avg_ms": 306.071, "count": 98, "max_ms": 10128.645, "p95_ms": 121.076}, "triton_wait": {"avg_ms": 70.048, "count": 98, "max_ms": 77.977, "p95_ms": 76.727}, "visualize": {"avg_ms": 3.378, "count": 98, "max_ms": 3.648, "p95_ms": 3.589}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10050}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10428.043, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:20:30+09:00", "last_error_summary": "[h264 @ 0x55dc1e599280] top block unavailable for requested intra mode -1 [h264 @ 0x55dc1e599280] error while decoding MB 58 0, bytestream 8703 [h264 @ 0x55dc1eec9400] left block unavailable for requested intra mode [h264 @ 0x55dc1eec9400] error while decoding MB 0 9, bytestream 26724 [h264 @ 0x55dc1e672ac0] error while decoding MB 9 2, bytestream -10 [in#0/rtsp @ 0x55dc1e513e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dc1e513e00] Error retrieving a packet from demuxer: Conn", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 355, "h264_decode_error_count": 1147, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:20:30+09:00", "last_error_summary": "[h264 @ 0x55dc1e599280] top block unavailable for requested intra mode -1 [h264 @ 0x55dc1e599280] error while decoding MB 58 0, bytestream 8703 [h264 @ 0x55dc1eec9400] left block unavailable for requested intra mode [h264 @ 0x55dc1eec9400] error while decoding MB 0 9, bytestream 26724 [h264 @ 0x55dc1e672ac0] error while decoding MB 9 2, bytestream -10 [in#0/rtsp @ 0x55dc1e513e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dc1e513e00] Error retrieving a packet from demuxer: Conn", "last_exit_at": "2026-07-01T01:20:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.011, "last_started_at": "2026-07-01T01:16:08+09:00", "pid": 70557, "process": {"exists": false, "pid": 70557}, "restart_count": 388, "restart_count_per_hour": 5, "rtsp_timeout_count": 328, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 389}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.309, "sample_count": 2336, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1206081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:20:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T01:20:30+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-01T01:20:30+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T01:20:30.566734+09:00", "mtime_epoch": 1782836430.566734, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121355}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:20:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4939.57, "name": "celery", "pid": 267, "rss_mb": 4939.57, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.448, "count": 99, "max_ms": 28.568, "p95_ms": 25.203}, "gstreamer_push": {"avg_ms": 1.221, "count": 99, "max_ms": 1.421, "p95_ms": 1.309}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.647, "count": 99, "max_ms": 44.679, "p95_ms": 41.505}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.398, "count": 99, "max_ms": 68.969, "p95_ms": 67.974}, "yolo11_640_fire_11class": {"avg_ms": 61.043, "count": 99, "max_ms": 70.018, "p95_ms": 66.828}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.783, "count": 99, "max_ms": 77.964, "p95_ms": 76.715}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.383, "count": 99, "max_ms": 106.838, "p95_ms": 105.016}, "read_interval": {"avg_ms": 306.071, "count": 98, "max_ms": 10128.645, "p95_ms": 121.076}, "triton_wait": {"avg_ms": 69.795, "count": 99, "max_ms": 77.977, "p95_ms": 76.727}, "visualize": {"avg_ms": 3.415, "count": 99, "max_ms": 7.048, "p95_ms": 3.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10050}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10428.156, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 355, "h264_decode_error_count": 1147, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:20:30+09:00", "last_error_summary": "[h264 @ 0x55dc1e599280] top block unavailable for requested intra mode -1 [h264 @ 0x55dc1e599280] error while decoding MB 58 0, bytestream 8703 [h264 @ 0x55dc1eec9400] left block unavailable for requested intra mode [h264 @ 0x55dc1eec9400] error while decoding MB 0 9, bytestream 26724 [h264 @ 0x55dc1e672ac0] error while decoding MB 9 2, bytestream -10 [in#0/rtsp @ 0x55dc1e513e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dc1e513e00] Error retrieving a packet from demuxer: Conn", "last_exit_at": "2026-07-01T01:20:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.019, "last_started_at": "2026-07-01T01:16:08+09:00", "pid": 70557, "process": {"exists": false, "pid": 70557}, "restart_count": 388, "restart_count_per_hour": 5, "rtsp_timeout_count": 328, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 389}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.309, "sample_count": 2335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1206081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:20:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T01:20:30+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-01T01:20:30+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-01T01:20:30.566734+09:00", "mtime_epoch": 1782836430.566734, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121355}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:20:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4939.57, "name": "celery", "pid": 267, "rss_mb": 4939.57, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.267, "pushed": 3.3, "read": 3.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.462, "count": 98, "max_ms": 28.568, "p95_ms": 25.203}, "gstreamer_push": {"avg_ms": 1.221, "count": 99, "max_ms": 1.421, "p95_ms": 1.309}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.667, "count": 98, "max_ms": 44.679, "p95_ms": 41.505}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.434, "count": 98, "max_ms": 68.969, "p95_ms": 67.974}, "yolo11_640_fire_11class": {"avg_ms": 61.055, "count": 98, "max_ms": 70.018, "p95_ms": 66.828}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.776, "count": 98, "max_ms": 77.964, "p95_ms": 76.715}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.395, "count": 98, "max_ms": 106.838, "p95_ms": 105.016}, "read_interval": {"avg_ms": 306.071, "count": 98, "max_ms": 10128.645, "p95_ms": 121.076}, "triton_wait": {"avg_ms": 69.789, "count": 98, "max_ms": 77.977, "p95_ms": 76.727}, "visualize": {"avg_ms": 3.416, "count": 98, "max_ms": 7.048, "p95_ms": 3.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10050}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10534.98, "limit_mb": 8796093022207.996}, "detail": {"pid": 70816, "restart_count": 389, "start_count": 390}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 355, "h264_decode_error_count": 1147, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:20:30+09:00", "last_error_summary": "[h264 @ 0x55dc1e599280] top block unavailable for requested intra mode -1 [h264 @ 0x55dc1e599280] error while decoding MB 58 0, bytestream 8703 [h264 @ 0x55dc1eec9400] left block unavailable for requested intra mode [h264 @ 0x55dc1eec9400] error while decoding MB 0 9, bytestream 26724 [h264 @ 0x55dc1e672ac0] error while decoding MB 9 2, bytestream -10 [in#0/rtsp @ 0x55dc1e513e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dc1e513e00] Error retrieving a packet from demuxer: Conn", "last_exit_at": "2026-07-01T01:20:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:20:32+09:00", "pid": 70816, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.121, "name": "ffmpeg", "pid": 70816, "rss_mb": 139.121, "state": "S (sleeping)", "threads": 106}, "restart_count": 389, "restart_count_per_hour": 6, "rtsp_timeout_count": 328, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 390}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.339, "sample_count": 2300, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1206081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:20:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.174, "last_frame_pushed_at": "2026-07-01T01:20:30+09:00", "last_frame_read_age_sec": 4.262, "last_frame_read_at": "2026-07-01T01:20:30+09:00", "mjpeg": {"age_sec": 4.152, "exists": true, "mtime": "2026-07-01T01:20:30.670736+09:00", "mtime_epoch": 1782836430.6707363, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121671}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:20:34+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4939.57, "name": "celery", "pid": 267, "rss_mb": 4939.57, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 62, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.067, "pushed": 2.067, "read": 2.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.787, "count": 62, "max_ms": 28.568, "p95_ms": 25.399}, "gstreamer_push": {"avg_ms": 1.234, "count": 62, "max_ms": 1.421, "p95_ms": 1.309}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.652, "count": 62, "max_ms": 44.679, "p95_ms": 40.893}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.856, "count": 62, "max_ms": 68.969, "p95_ms": 67.934}, "yolo11_640_fire_11class": {"avg_ms": 59.859, "count": 62, "max_ms": 67.425, "p95_ms": 66.784}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.096, "count": 62, "max_ms": 77.964, "p95_ms": 76.357}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.102, "count": 62, "max_ms": 106.838, "p95_ms": 104.892}, "read_interval": {"avg_ms": 416.885, "count": 62, "max_ms": 10128.645, "p95_ms": 121.914}, "triton_wait": {"avg_ms": 68.109, "count": 62, "max_ms": 77.977, "p95_ms": 76.37}, "visualize": {"avg_ms": 3.458, "count": 62, "max_ms": 7.048, "p95_ms": 3.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11246}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10365.336, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559ec80e5040] error while decoding MB 70 29, bytestream -5 [in#0/rtsp @ 0x559ec7758e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ec7758e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 158, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:21:38+09:00", "last_error_summary": "[h264 @ 0x559ec80e5040] error while decoding MB 70 29, bytestream -5 [in#0/rtsp @ 0x559ec7758e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ec7758e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:17:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 266.397, "last_started_at": "2026-07-01T01:17:12+09:00", "pid": 70664, "process": {"exists": false, "pid": 70664}, "restart_count": 152, "restart_count_per_hour": 9, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 153}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.58, "sample_count": 2137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349564, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:21:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-01T01:21:38+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-01T01:21:38+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T01:21:38.856119+09:00", "mtime_epoch": 1782836498.8561194, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90362}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:21:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1697.098, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.738, "pushed": 0.738, "read": 0.795}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.348, "count": 13, "max_ms": 18.792, "p95_ms": 18.792}, "gstreamer_push": {"avg_ms": 1.253, "count": 13, "max_ms": 1.398, "p95_ms": 1.398}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.886, "count": 13, "max_ms": 67.366, "p95_ms": 67.366}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.582, "count": 13, "max_ms": 68.463, "p95_ms": 68.463}, "yolo11_640_fire_11class": {"avg_ms": 63.181, "count": 13, "max_ms": 67.965, "p95_ms": 67.965}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.672, "count": 13, "max_ms": 77.116, "p95_ms": 77.116}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.169, "count": 13, "max_ms": 102.683, "p95_ms": 102.683}, "read_interval": {"avg_ms": 2395.642, "count": 14, "max_ms": 16190.587, "p95_ms": 16190.587}, "triton_wait": {"avg_ms": 71.685, "count": 13, "max_ms": 77.128, "p95_ms": 77.128}, "visualize": {"avg_ms": 3.412, "count": 13, "max_ms": 3.512, "p95_ms": 3.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.606, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11246}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10364.836, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:21:38+09:00", "last_error_summary": "[h264 @ 0x559ec80e5040] error while decoding MB 70 29, bytestream -5 [in#0/rtsp @ 0x559ec7758e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ec7758e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 158, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:21:38+09:00", "last_error_summary": "[h264 @ 0x559ec80e5040] error while decoding MB 70 29, bytestream -5 [in#0/rtsp @ 0x559ec7758e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ec7758e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:21:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 266.442, "last_started_at": "2026-07-01T01:17:12+09:00", "pid": 70664, "process": {"exists": false, "pid": 70664}, "restart_count": 152, "restart_count_per_hour": 9, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 153}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.58, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349565, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:21:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T01:21:38+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-01T01:21:38+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-01T01:21:38.856119+09:00", "mtime_epoch": 1782836498.8561194, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90362}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:21:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1697.098, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.793, "pushed": 0.793, "read": 0.793}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.417, "count": 14, "max_ms": 19.31, "p95_ms": 19.31}, "gstreamer_push": {"avg_ms": 1.25, "count": 14, "max_ms": 1.398, "p95_ms": 1.398}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.742, "count": 14, "max_ms": 67.366, "p95_ms": 67.366}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.862, "count": 14, "max_ms": 68.496, "p95_ms": 68.496}, "yolo11_640_fire_11class": {"avg_ms": 63.517, "count": 14, "max_ms": 67.965, "p95_ms": 67.965}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.772, "count": 14, "max_ms": 77.116, "p95_ms": 77.116}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.338, "count": 14, "max_ms": 102.683, "p95_ms": 102.683}, "read_interval": {"avg_ms": 2395.642, "count": 14, "max_ms": 16190.587, "p95_ms": 16190.587}, "triton_wait": {"avg_ms": 71.785, "count": 14, "max_ms": 77.128, "p95_ms": 77.128}, "visualize": {"avg_ms": 3.416, "count": 14, "max_ms": 3.512, "p95_ms": 3.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.651, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11246}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10365.105, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 158, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:21:38+09:00", "last_error_summary": "[h264 @ 0x559ec80e5040] error while decoding MB 70 29, bytestream -5 [in#0/rtsp @ 0x559ec7758e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ec7758e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:21:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 266.448, "last_started_at": "2026-07-01T01:17:12+09:00", "pid": 70664, "process": {"exists": false, "pid": 70664}, "restart_count": 152, "restart_count_per_hour": 9, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 153}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.58, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349565, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:21:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T01:21:38+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T01:21:38+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-01T01:21:38.856119+09:00", "mtime_epoch": 1782836498.8561194, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90362}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:21:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1697.098, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.793, "pushed": 0.793, "read": 0.793}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.417, "count": 14, "max_ms": 19.31, "p95_ms": 19.31}, "gstreamer_push": {"avg_ms": 1.25, "count": 14, "max_ms": 1.398, "p95_ms": 1.398}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.742, "count": 14, "max_ms": 67.366, "p95_ms": 67.366}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.862, "count": 14, "max_ms": 68.496, "p95_ms": 68.496}, "yolo11_640_fire_11class": {"avg_ms": 63.517, "count": 14, "max_ms": 67.965, "p95_ms": 67.965}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.772, "count": 14, "max_ms": 77.116, "p95_ms": 77.116}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.338, "count": 14, "max_ms": 102.683, "p95_ms": 102.683}, "read_interval": {"avg_ms": 2395.642, "count": 14, "max_ms": 16190.587, "p95_ms": 16190.587}, "triton_wait": {"avg_ms": 71.785, "count": 14, "max_ms": 77.128, "p95_ms": 77.128}, "visualize": {"avg_ms": 3.416, "count": 14, "max_ms": 3.512, "p95_ms": 3.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.657, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11246}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10489.008, "limit_mb": 8796093022207.996}, "detail": {"pid": 70923, "restart_count": 153, "start_count": 154}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 158, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:21:38+09:00", "last_error_summary": "[h264 @ 0x559ec80e5040] error while decoding MB 70 29, bytestream -5 [in#0/rtsp @ 0x559ec7758e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ec7758e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:21:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:21:41+09:00", "pid": 70923, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.367, "name": "ffmpeg", "pid": 70923, "rss_mb": 156.203, "state": "R (running)", "threads": 49}, "restart_count": 153, "restart_count_per_hour": 10, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 154}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.58, "sample_count": 2111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349565, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:21:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.142, "last_frame_pushed_at": "2026-07-01T01:21:38+09:00", "last_frame_read_age_sec": 4.245, "last_frame_read_at": "2026-07-01T01:21:38+09:00", "mjpeg": {"age_sec": 4.13, "exists": true, "mtime": "2026-07-01T01:21:38.968122+09:00", "mtime_epoch": 1782836498.9681218, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89750}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:21:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1697.098, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.643, "pushed": 0.643, "read": 0.643}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.417, "count": 14, "max_ms": 19.31, "p95_ms": 19.31}, "gstreamer_push": {"avg_ms": 1.25, "count": 14, "max_ms": 1.398, "p95_ms": 1.398}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.742, "count": 14, "max_ms": 67.366, "p95_ms": 67.366}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.862, "count": 14, "max_ms": 68.496, "p95_ms": 68.496}, "yolo11_640_fire_11class": {"avg_ms": 63.517, "count": 14, "max_ms": 67.965, "p95_ms": 67.965}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.772, "count": 14, "max_ms": 77.116, "p95_ms": 77.116}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.338, "count": 14, "max_ms": 102.683, "p95_ms": 102.683}, "read_interval": {"avg_ms": 2395.642, "count": 14, "max_ms": 16190.587, "p95_ms": 16190.587}, "triton_wait": {"avg_ms": 71.785, "count": 14, "max_ms": 77.128, "p95_ms": 77.128}, "visualize": {"avg_ms": 3.416, "count": 14, "max_ms": 3.512, "p95_ms": 3.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.789, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10429.332, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563b830eb740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830eb740] error while decoding MB 0 18, bytestream 1370 [h264 @ 0x563b83ca42c0] left block unavailable for requested intra mode [h264 @ 0x563b83ca42c0] error while decoding MB 0 20, bytestream 6115 [h264 @ 0x563b830ff840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830ff840] error while decoding MB 0 2, bytestream 80 [h264 @ 0x563b830c0440] top block unavailable for requested intra", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 356, "h264_decode_error_count": 1156, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:27:04+09:00", "last_error_summary": "[h264 @ 0x563b830eb740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830eb740] error while decoding MB 0 18, bytestream 1370 [h264 @ 0x563b83ca42c0] left block unavailable for requested intra mode [h264 @ 0x563b83ca42c0] error while decoding MB 0 20, bytestream 6115 [h264 @ 0x563b830ff840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830ff840] error while decoding MB 0 2, bytestream 80 [h264 @ 0x563b830c0440] top block unavailable for requested intra", "last_exit_at": "2026-07-01T01:20:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 392.007, "last_started_at": "2026-07-01T01:20:32+09:00", "pid": 70816, "process": {"exists": false, "pid": 70816}, "restart_count": 389, "restart_count_per_hour": 6, "rtsp_timeout_count": 330, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 390}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.374, "sample_count": 2520, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1209478, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:27:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-01T01:27:04+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-01T01:27:04+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-01T01:27:04.778500+09:00", "mtime_epoch": 1782836824.7784998, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 135305}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:27:04+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4941.375, "name": "celery", "pid": 267, "rss_mb": 4941.375, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.306, "pushed": 4.306, "read": 4.306}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.407, "count": 129, "max_ms": 24.232, "p95_ms": 23.149}, "gstreamer_push": {"avg_ms": 1.233, "count": 129, "max_ms": 1.358, "p95_ms": 1.326}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.873, "count": 129, "max_ms": 44.352, "p95_ms": 41.929}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.814, "count": 129, "max_ms": 74.065, "p95_ms": 68.71}, "yolo11_640_fire_11class": {"avg_ms": 59.982, "count": 129, "max_ms": 72.907, "p95_ms": 67.313}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.438, "count": 129, "max_ms": 82.702, "p95_ms": 77.738}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.925, "count": 129, "max_ms": 111.054, "p95_ms": 106.156}, "read_interval": {"avg_ms": 232.409, "count": 129, "max_ms": 15538.095, "p95_ms": 122.691}, "triton_wait": {"avg_ms": 68.467, "count": 129, "max_ms": 82.715, "p95_ms": 77.75}, "visualize": {"avg_ms": 3.379, "count": 129, "max_ms": 3.97, "p95_ms": 3.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10428.824, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:27:04+09:00", "last_error_summary": "[h264 @ 0x563b830eb740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830eb740] error while decoding MB 0 18, bytestream 1370 [h264 @ 0x563b83ca42c0] left block unavailable for requested intra mode [h264 @ 0x563b83ca42c0] error while decoding MB 0 20, bytestream 6115 [h264 @ 0x563b830ff840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830ff840] error while decoding MB 0 2, bytestream 80 [h264 @ 0x563b830c0440] top block unavailable for requested intra", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 356, "h264_decode_error_count": 1156, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:27:04+09:00", "last_error_summary": "[h264 @ 0x563b830eb740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830eb740] error while decoding MB 0 18, bytestream 1370 [h264 @ 0x563b83ca42c0] left block unavailable for requested intra mode [h264 @ 0x563b83ca42c0] error while decoding MB 0 20, bytestream 6115 [h264 @ 0x563b830ff840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830ff840] error while decoding MB 0 2, bytestream 80 [h264 @ 0x563b830c0440] top block unavailable for requested intra", "last_exit_at": "2026-07-01T01:27:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 392.065, "last_started_at": "2026-07-01T01:20:32+09:00", "pid": 70816, "process": {"exists": false, "pid": 70816}, "restart_count": 389, "restart_count_per_hour": 6, "rtsp_timeout_count": 330, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 390}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.374, "sample_count": 2520, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1209479, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:27:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T01:27:04+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-01T01:27:04+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-01T01:27:04.778500+09:00", "mtime_epoch": 1782836824.7784998, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 135305}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:27:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4941.375, "name": "celery", "pid": 267, "rss_mb": 4941.375, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.31, "pushed": 4.31, "read": 4.277}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.389, "count": 129, "max_ms": 24.232, "p95_ms": 23.149}, "gstreamer_push": {"avg_ms": 1.233, "count": 129, "max_ms": 1.358, "p95_ms": 1.326}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.895, "count": 129, "max_ms": 44.352, "p95_ms": 41.929}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.982, "count": 129, "max_ms": 74.065, "p95_ms": 68.71}, "yolo11_640_fire_11class": {"avg_ms": 60.159, "count": 129, "max_ms": 72.907, "p95_ms": 67.313}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.681, "count": 129, "max_ms": 82.702, "p95_ms": 77.738}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.153, "count": 129, "max_ms": 111.054, "p95_ms": 106.228}, "read_interval": {"avg_ms": 233.51, "count": 128, "max_ms": 15538.095, "p95_ms": 122.691}, "triton_wait": {"avg_ms": 68.71, "count": 129, "max_ms": 82.715, "p95_ms": 77.75}, "visualize": {"avg_ms": 3.38, "count": 129, "max_ms": 3.97, "p95_ms": 3.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10428.723, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 356, "h264_decode_error_count": 1156, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:27:04+09:00", "last_error_summary": "[h264 @ 0x563b830eb740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830eb740] error while decoding MB 0 18, bytestream 1370 [h264 @ 0x563b83ca42c0] left block unavailable for requested intra mode [h264 @ 0x563b83ca42c0] error while decoding MB 0 20, bytestream 6115 [h264 @ 0x563b830ff840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830ff840] error while decoding MB 0 2, bytestream 80 [h264 @ 0x563b830c0440] top block unavailable for requested intra", "last_exit_at": "2026-07-01T01:27:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 392.072, "last_started_at": "2026-07-01T01:20:32+09:00", "pid": 70816, "process": {"exists": false, "pid": 70816}, "restart_count": 389, "restart_count_per_hour": 6, "rtsp_timeout_count": 330, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 390}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.374, "sample_count": 2520, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1209479, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:27:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T01:27:04+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-01T01:27:04+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-01T01:27:04.778500+09:00", "mtime_epoch": 1782836824.7784998, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 135305}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:27:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4941.375, "name": "celery", "pid": 267, "rss_mb": 4941.375, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.309, "pushed": 4.309, "read": 4.276}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.389, "count": 129, "max_ms": 24.232, "p95_ms": 23.149}, "gstreamer_push": {"avg_ms": 1.233, "count": 129, "max_ms": 1.358, "p95_ms": 1.326}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.895, "count": 129, "max_ms": 44.352, "p95_ms": 41.929}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.982, "count": 129, "max_ms": 74.065, "p95_ms": 68.71}, "yolo11_640_fire_11class": {"avg_ms": 60.159, "count": 129, "max_ms": 72.907, "p95_ms": 67.313}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.681, "count": 129, "max_ms": 82.702, "p95_ms": 77.738}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.153, "count": 129, "max_ms": 111.054, "p95_ms": 106.228}, "read_interval": {"avg_ms": 233.51, "count": 128, "max_ms": 15538.095, "p95_ms": 122.691}, "triton_wait": {"avg_ms": 68.71, "count": 129, "max_ms": 82.715, "p95_ms": 77.75}, "visualize": {"avg_ms": 3.38, "count": 129, "max_ms": 3.97, "p95_ms": 3.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10536.938, "limit_mb": 8796093022207.996}, "detail": {"pid": 71075, "restart_count": 390, "start_count": 391}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 356, "h264_decode_error_count": 1156, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:27:04+09:00", "last_error_summary": "[h264 @ 0x563b830eb740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830eb740] error while decoding MB 0 18, bytestream 1370 [h264 @ 0x563b83ca42c0] left block unavailable for requested intra mode [h264 @ 0x563b83ca42c0] error while decoding MB 0 20, bytestream 6115 [h264 @ 0x563b830ff840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563b830ff840] error while decoding MB 0 2, bytestream 80 [h264 @ 0x563b830c0440] top block unavailable for requested intra", "last_exit_at": "2026-07-01T01:27:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:27:07+09:00", "pid": 71075, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.281, "name": "ffmpeg", "pid": 71075, "rss_mb": 139.281, "state": "S (sleeping)", "threads": 106}, "restart_count": 390, "restart_count_per_hour": 7, "rtsp_timeout_count": 330, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 391}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.382, "sample_count": 2482, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1209479, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:27:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.541, "last_frame_pushed_at": "2026-07-01T01:27:04+09:00", "last_frame_read_age_sec": 4.649, "last_frame_read_at": "2026-07-01T01:27:04+09:00", "mjpeg": {"age_sec": 4.523, "exists": true, "mtime": "2026-07-01T01:27:04.898502+09:00", "mtime_epoch": 1782836824.898502, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 133835}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:27:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4941.375, "name": "celery", "pid": 267, "rss_mb": 4941.375, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.873, "pushed": 2.873, "read": 2.839}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.045, "count": 86, "max_ms": 21.534, "p95_ms": 20.684}, "gstreamer_push": {"avg_ms": 1.232, "count": 86, "max_ms": 1.348, "p95_ms": 1.315}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.981, "count": 86, "max_ms": 44.352, "p95_ms": 42.762}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.064, "count": 86, "max_ms": 72.79, "p95_ms": 69.808}, "yolo11_640_fire_11class": {"avg_ms": 63.113, "count": 86, "max_ms": 72.181, "p95_ms": 68.17}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.081, "count": 86, "max_ms": 82.092, "p95_ms": 78.431}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.199, "count": 86, "max_ms": 110.65, "p95_ms": 106.228}, "read_interval": {"avg_ms": 298.746, "count": 85, "max_ms": 15538.095, "p95_ms": 122.691}, "triton_wait": {"avg_ms": 73.119, "count": 86, "max_ms": 82.101, "p95_ms": 78.444}, "visualize": {"avg_ms": 3.361, "count": 86, "max_ms": 3.97, "p95_ms": 3.503}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11271}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10667.695, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 158, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:21:38+09:00", "last_error_summary": "[h264 @ 0x559ec80e5040] error while decoding MB 70 29, bytestream -5 [in#0/rtsp @ 0x559ec7758e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ec7758e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:21:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 365.557, "last_started_at": "2026-07-01T01:21:41+09:00", "pid": 70923, "process": {"exists": true, "fd_count": 5, "hwm_mb": 324.742, "name": "ffmpeg", "pid": 70923, "rss_mb": 324.742, "state": "S (sleeping)", "threads": 106}, "restart_count": 153, "restart_count_per_hour": 9, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 154}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.217, "sample_count": 2451, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1352563, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:27:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.057, "last_frame_pushed_at": "2026-07-01T01:27:16+09:00", "last_frame_read_age_sec": 30.163, "last_frame_read_at": "2026-07-01T01:27:16+09:00", "mjpeg": {"age_sec": 30.039, "exists": true, "mtime": "2026-07-01T01:27:16.614726+09:00", "mtime_epoch": 1782836836.614726, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93261}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:27:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1698.387, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11271}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10659.477, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 158, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:21:38+09:00", "last_error_summary": "[h264 @ 0x559ec80e5040] error while decoding MB 70 29, bytestream -5 [in#0/rtsp @ 0x559ec7758e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559ec7758e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:21:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 365.569, "last_started_at": "2026-07-01T01:21:41+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 153, "restart_count_per_hour": 9, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 154}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.217, "sample_count": 2451, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1352563, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:27:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.07, "last_frame_pushed_at": "2026-07-01T01:27:16+09:00", "last_frame_read_age_sec": 30.175, "last_frame_read_at": "2026-07-01T01:27:16+09:00", "mjpeg": {"age_sec": 30.052, "exists": true, "mtime": "2026-07-01T01:27:16.614726+09:00", "mtime_epoch": 1782836836.614726, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93261}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:27:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1698.387, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11271}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10370.203, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[NULL @ 0x55a9267f6200] missing picture in access unit with size 95 [h264 @ 0x55a9270b17c0] no frame!", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 154, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:27:46+09:00", "last_error_summary": "[NULL @ 0x55a9267f6200] missing picture in access unit with size 95 [h264 @ 0x55a9270b17c0] no frame!", "last_exit_at": "2026-07-01T01:21:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 365.619, "last_started_at": "2026-07-01T01:21:41+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 153, "restart_count_per_hour": 9, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 154}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.217, "sample_count": 2450, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1352563, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:27:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.119, "last_frame_pushed_at": "2026-07-01T01:27:16+09:00", "last_frame_read_age_sec": 30.225, "last_frame_read_at": "2026-07-01T01:27:16+09:00", "mjpeg": {"age_sec": 30.101, "exists": true, "mtime": "2026-07-01T01:27:16.614726+09:00", "mtime_epoch": 1782836836.614726, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93261}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:27:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1698.387, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11271}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10546.301, "limit_mb": 8796093022207.996}, "detail": {"pid": 71182, "restart_count": 154, "start_count": 155}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 154, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:27:46+09:00", "last_error_summary": "[NULL @ 0x55a9267f6200] missing picture in access unit with size 95 [h264 @ 0x55a9270b17c0] no frame!", "last_exit_at": "2026-07-01T01:21:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:27:49+09:00", "pid": 71182, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.711, "name": "ffmpeg", "pid": 71182, "rss_mb": 206.711, "state": "S (sleeping)", "threads": 106}, "restart_count": 154, "restart_count_per_hour": 10, "rtsp_timeout_count": 248, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 155}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.265, "sample_count": 2411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1352563, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:27:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 34.655, "last_frame_pushed_at": "2026-07-01T01:27:16+09:00", "last_frame_read_age_sec": 34.761, "last_frame_read_at": "2026-07-01T01:27:16+09:00", "mjpeg": {"age_sec": 34.637, "exists": true, "mtime": "2026-07-01T01:27:16.614726+09:00", "mtime_epoch": 1782836836.614726, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93261}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:27:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1698.387, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11286}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10398.582, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563d3fb33180] error while decoding MB 98 16, bytestream -5 [in#0/rtsp @ 0x563d3faabe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563d3faabe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 155, "h264_decode_error_count": 160, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:31:42+09:00", "last_error_summary": "[h264 @ 0x563d3fb33180] error while decoding MB 98 16, bytestream -5 [in#0/rtsp @ 0x563d3faabe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563d3faabe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:21:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.16, "last_started_at": "2026-07-01T01:27:49+09:00", "pid": 71182, "process": {"exists": false, "pid": 71182}, "restart_count": 154, "restart_count_per_hour": 10, "rtsp_timeout_count": 250, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 155}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.698, "sample_count": 2148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1354366, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:31:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T01:31:42+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-01T01:31:42+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T01:31:42.347736+09:00", "mtime_epoch": 1782837102.347736, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95758}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:31:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1699.418, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.754, "pushed": 0.754, "read": 0.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.17, "count": 13, "max_ms": 21.348, "p95_ms": 21.348}, "gstreamer_push": {"avg_ms": 1.215, "count": 13, "max_ms": 1.474, "p95_ms": 1.474}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.973, "count": 13, "max_ms": 46.836, "p95_ms": 46.836}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.699, "count": 13, "max_ms": 50.464, "p95_ms": 50.464}, "yolo11_640_fire_11class": {"avg_ms": 47.518, "count": 13, "max_ms": 51.149, "p95_ms": 51.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.59, "count": 13, "max_ms": 51.753, "p95_ms": 51.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.921, "count": 13, "max_ms": 79.911, "p95_ms": 79.911}, "read_interval": {"avg_ms": 2583.932, "count": 14, "max_ms": 18995.436, "p95_ms": 18995.436}, "triton_wait": {"avg_ms": 48.602, "count": 13, "max_ms": 51.762, "p95_ms": 51.762}, "visualize": {"avg_ms": 3.42, "count": 13, "max_ms": 3.566, "p95_ms": 3.566}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.249, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11286}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10398.348, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:31:42+09:00", "last_error_summary": "[h264 @ 0x563d3fb33180] error while decoding MB 98 16, bytestream -5 [in#0/rtsp @ 0x563d3faabe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563d3faabe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 155, "h264_decode_error_count": 160, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:31:42+09:00", "last_error_summary": "[h264 @ 0x563d3fb33180] error while decoding MB 98 16, bytestream -5 [in#0/rtsp @ 0x563d3faabe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563d3faabe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:31:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.186, "last_started_at": "2026-07-01T01:27:49+09:00", "pid": 71182, "process": {"exists": false, "pid": 71182}, "restart_count": 154, "restart_count_per_hour": 10, "rtsp_timeout_count": 250, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 155}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.698, "sample_count": 2148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1354367, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:31:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T01:31:42+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-01T01:31:42+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-01T01:31:42.347736+09:00", "mtime_epoch": 1782837102.347736, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95758}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:31:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1699.418, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.81, "pushed": 0.81, "read": 0.81}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.372, "count": 14, "max_ms": 22.996, "p95_ms": 22.996}, "gstreamer_push": {"avg_ms": 1.226, "count": 14, "max_ms": 1.474, "p95_ms": 1.474}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.961, "count": 14, "max_ms": 46.836, "p95_ms": 46.836}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.844, "count": 14, "max_ms": 50.464, "p95_ms": 50.464}, "yolo11_640_fire_11class": {"avg_ms": 47.635, "count": 14, "max_ms": 51.149, "p95_ms": 51.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.74, "count": 14, "max_ms": 51.753, "p95_ms": 51.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.495, "count": 14, "max_ms": 84.958, "p95_ms": 84.958}, "read_interval": {"avg_ms": 2583.932, "count": 14, "max_ms": 18995.436, "p95_ms": 18995.436}, "triton_wait": {"avg_ms": 48.751, "count": 14, "max_ms": 51.762, "p95_ms": 51.762}, "visualize": {"avg_ms": 3.623, "count": 14, "max_ms": 6.257, "p95_ms": 6.257}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.275, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11286}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10398.34, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 155, "h264_decode_error_count": 160, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:31:42+09:00", "last_error_summary": "[h264 @ 0x563d3fb33180] error while decoding MB 98 16, bytestream -5 [in#0/rtsp @ 0x563d3faabe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563d3faabe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:31:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.193, "last_started_at": "2026-07-01T01:27:49+09:00", "pid": 71182, "process": {"exists": false, "pid": 71182}, "restart_count": 154, "restart_count_per_hour": 10, "rtsp_timeout_count": 250, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 155}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.698, "sample_count": 2148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1354367, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:31:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T01:31:42+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T01:31:42+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-01T01:31:42.347736+09:00", "mtime_epoch": 1782837102.347736, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95758}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:31:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1699.418, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.81, "pushed": 0.81, "read": 0.81}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.372, "count": 14, "max_ms": 22.996, "p95_ms": 22.996}, "gstreamer_push": {"avg_ms": 1.226, "count": 14, "max_ms": 1.474, "p95_ms": 1.474}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.961, "count": 14, "max_ms": 46.836, "p95_ms": 46.836}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.844, "count": 14, "max_ms": 50.464, "p95_ms": 50.464}, "yolo11_640_fire_11class": {"avg_ms": 47.635, "count": 14, "max_ms": 51.149, "p95_ms": 51.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.74, "count": 14, "max_ms": 51.753, "p95_ms": 51.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.495, "count": 14, "max_ms": 84.958, "p95_ms": 84.958}, "read_interval": {"avg_ms": 2583.932, "count": 14, "max_ms": 18995.436, "p95_ms": 18995.436}, "triton_wait": {"avg_ms": 48.751, "count": 14, "max_ms": 51.762, "p95_ms": 51.762}, "visualize": {"avg_ms": 3.623, "count": 14, "max_ms": 6.257, "p95_ms": 6.257}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.282, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11286}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10575.387, "limit_mb": 8796093022207.996}, "detail": {"pid": 71334, "restart_count": 155, "start_count": 156}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 155, "h264_decode_error_count": 160, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:31:42+09:00", "last_error_summary": "[h264 @ 0x563d3fb33180] error while decoding MB 98 16, bytestream -5 [in#0/rtsp @ 0x563d3faabe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563d3faabe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:31:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:31:45+09:00", "pid": 71334, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.801, "name": "ffmpeg", "pid": 71334, "rss_mb": 206.801, "state": "S (sleeping)", "threads": 106}, "restart_count": 155, "restart_count_per_hour": 11, "rtsp_timeout_count": 250, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 156}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.683, "sample_count": 2106, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1354367, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:31:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.983, "last_frame_pushed_at": "2026-07-01T01:31:42+09:00", "last_frame_read_age_sec": 5.07, "last_frame_read_at": "2026-07-01T01:31:42+09:00", "mjpeg": {"age_sec": 4.964, "exists": true, "mtime": "2026-07-01T01:31:42.447738+09:00", "mtime_epoch": 1782837102.4477377, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94854}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:31:47+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1699.418, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.629, "pushed": 0.629, "read": 0.629}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.372, "count": 14, "max_ms": 22.996, "p95_ms": 22.996}, "gstreamer_push": {"avg_ms": 1.226, "count": 14, "max_ms": 1.474, "p95_ms": 1.474}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.961, "count": 14, "max_ms": 46.836, "p95_ms": 46.836}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.844, "count": 14, "max_ms": 50.464, "p95_ms": 50.464}, "yolo11_640_fire_11class": {"avg_ms": 47.635, "count": 14, "max_ms": 51.149, "p95_ms": 51.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.74, "count": 14, "max_ms": 51.753, "p95_ms": 51.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.495, "count": 14, "max_ms": 84.958, "p95_ms": 84.958}, "read_interval": {"avg_ms": 2583.932, "count": 14, "max_ms": 18995.436, "p95_ms": 18995.436}, "triton_wait": {"avg_ms": 48.751, "count": 14, "max_ms": 51.762, "p95_ms": 51.762}, "visualize": {"avg_ms": 3.623, "count": 14, "max_ms": 6.257, "p95_ms": 6.257}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.25, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11293}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10402.477, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55941fc91a40] error while decoding MB 101 34, bytestream -5 [in#0/rtsp @ 0x55941f084e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55941f084e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 156, "h264_decode_error_count": 161, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:34:00+09:00", "last_error_summary": "[h264 @ 0x55941fc91a40] error while decoding MB 101 34, bytestream -5 [in#0/rtsp @ 0x55941f084e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55941f084e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:31:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.879, "last_started_at": "2026-07-01T01:31:45+09:00", "pid": 71334, "process": {"exists": false, "pid": 71334}, "restart_count": 155, "restart_count_per_hour": 11, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 156}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.369, "sample_count": 2110, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355271, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:34:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-01T01:34:00+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-01T01:34:00+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-01T01:34:00.234296+09:00", "mtime_epoch": 1782837240.2342956, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96293}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:34:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1699.676, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.734, "pushed": 0.734, "read": 0.79}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.29, "count": 13, "max_ms": 19.165, "p95_ms": 19.165}, "gstreamer_push": {"avg_ms": 1.231, "count": 13, "max_ms": 1.307, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.043, "count": 13, "max_ms": 56.881, "p95_ms": 56.881}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.055, "count": 13, "max_ms": 65.793, "p95_ms": 65.793}, "yolo11_640_fire_11class": {"avg_ms": 63.211, "count": 13, "max_ms": 65.237, "p95_ms": 65.237}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.828, "count": 13, "max_ms": 74.663, "p95_ms": 74.663}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.179, "count": 13, "max_ms": 101.481, "p95_ms": 101.481}, "read_interval": {"avg_ms": 2297.041, "count": 14, "max_ms": 16322.931, "p95_ms": 16322.931}, "triton_wait": {"avg_ms": 71.84, "count": 13, "max_ms": 74.675, "p95_ms": 74.675}, "visualize": {"avg_ms": 3.377, "count": 13, "max_ms": 3.458, "p95_ms": 3.458}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.72, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11293}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10402.289, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:34:00+09:00", "last_error_summary": "[h264 @ 0x55941fc91a40] error while decoding MB 101 34, bytestream -5 [in#0/rtsp @ 0x55941f084e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55941f084e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 156, "h264_decode_error_count": 161, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:34:00+09:00", "last_error_summary": "[h264 @ 0x55941fc91a40] error while decoding MB 101 34, bytestream -5 [in#0/rtsp @ 0x55941f084e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55941f084e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:34:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.921, "last_started_at": "2026-07-01T01:31:45+09:00", "pid": 71334, "process": {"exists": false, "pid": 71334}, "restart_count": 155, "restart_count_per_hour": 11, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 156}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.369, "sample_count": 2110, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:34:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T01:34:00+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T01:34:00+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-01T01:34:00.234296+09:00", "mtime_epoch": 1782837240.2342956, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96293}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:34:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1699.676, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.788, "pushed": 0.788, "read": 0.788}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.306, "count": 14, "max_ms": 19.165, "p95_ms": 19.165}, "gstreamer_push": {"avg_ms": 1.232, "count": 14, "max_ms": 1.307, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.726, "count": 14, "max_ms": 56.881, "p95_ms": 56.881}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.175, "count": 14, "max_ms": 65.793, "p95_ms": 65.793}, "yolo11_640_fire_11class": {"avg_ms": 63.393, "count": 14, "max_ms": 65.75, "p95_ms": 65.75}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.762, "count": 14, "max_ms": 74.663, "p95_ms": 74.663}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.138, "count": 14, "max_ms": 101.481, "p95_ms": 101.481}, "read_interval": {"avg_ms": 2297.041, "count": 14, "max_ms": 16322.931, "p95_ms": 16322.931}, "triton_wait": {"avg_ms": 71.774, "count": 14, "max_ms": 74.675, "p95_ms": 74.675}, "visualize": {"avg_ms": 3.399, "count": 14, "max_ms": 3.687, "p95_ms": 3.687}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.762, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11293}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10402.145, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 156, "h264_decode_error_count": 161, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:34:00+09:00", "last_error_summary": "[h264 @ 0x55941fc91a40] error while decoding MB 101 34, bytestream -5 [in#0/rtsp @ 0x55941f084e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55941f084e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:34:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.927, "last_started_at": "2026-07-01T01:31:45+09:00", "pid": 71334, "process": {"exists": false, "pid": 71334}, "restart_count": 155, "restart_count_per_hour": 11, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 156}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.369, "sample_count": 2110, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:34:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T01:34:00+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T01:34:00+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-01T01:34:00.234296+09:00", "mtime_epoch": 1782837240.2342956, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96293}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:34:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1699.676, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.788, "pushed": 0.788, "read": 0.788}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.306, "count": 14, "max_ms": 19.165, "p95_ms": 19.165}, "gstreamer_push": {"avg_ms": 1.232, "count": 14, "max_ms": 1.307, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.726, "count": 14, "max_ms": 56.881, "p95_ms": 56.881}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.175, "count": 14, "max_ms": 65.793, "p95_ms": 65.793}, "yolo11_640_fire_11class": {"avg_ms": 63.393, "count": 14, "max_ms": 65.75, "p95_ms": 65.75}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.762, "count": 14, "max_ms": 74.663, "p95_ms": 74.663}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.138, "count": 14, "max_ms": 101.481, "p95_ms": 101.481}, "read_interval": {"avg_ms": 2297.041, "count": 14, "max_ms": 16322.931, "p95_ms": 16322.931}, "triton_wait": {"avg_ms": 71.774, "count": 14, "max_ms": 74.675, "p95_ms": 74.675}, "visualize": {"avg_ms": 3.399, "count": 14, "max_ms": 3.687, "p95_ms": 3.687}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.767, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11293}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10422.941, "limit_mb": 8796093022207.996}, "detail": {"pid": 71486, "restart_count": 156, "start_count": 157}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 156, "h264_decode_error_count": 161, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:34:00+09:00", "last_error_summary": "[h264 @ 0x55941fc91a40] error while decoding MB 101 34, bytestream -5 [in#0/rtsp @ 0x55941f084e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55941f084e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:34:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:34:02+09:00", "pid": 71486, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.211, "name": "ffmpeg", "pid": 71486, "rss_mb": 58.211, "state": "S (sleeping)", "threads": 1}, "restart_count": 156, "restart_count_per_hour": 12, "rtsp_timeout_count": 252, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 157}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.369, "sample_count": 2072, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:34:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.499, "last_frame_pushed_at": "2026-07-01T01:34:00+09:00", "last_frame_read_age_sec": 4.599, "last_frame_read_at": "2026-07-01T01:34:00+09:00", "mjpeg": {"age_sec": 4.484, "exists": true, "mtime": "2026-07-01T01:34:00.342298+09:00", "mtime_epoch": 1782837240.3422976, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96147}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:34:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1699.676, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.629, "pushed": 0.629, "read": 0.629}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.306, "count": 14, "max_ms": 19.165, "p95_ms": 19.165}, "gstreamer_push": {"avg_ms": 1.232, "count": 14, "max_ms": 1.307, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.726, "count": 14, "max_ms": 56.881, "p95_ms": 56.881}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.175, "count": 14, "max_ms": 65.793, "p95_ms": 65.793}, "yolo11_640_fire_11class": {"avg_ms": 63.393, "count": 14, "max_ms": 65.75, "p95_ms": 65.75}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.762, "count": 14, "max_ms": 74.663, "p95_ms": 74.663}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.138, "count": 14, "max_ms": 101.481, "p95_ms": 101.481}, "read_interval": {"avg_ms": 2297.041, "count": 14, "max_ms": 16322.931, "p95_ms": 16322.931}, "triton_wait": {"avg_ms": 71.774, "count": 14, "max_ms": 74.675, "p95_ms": 74.675}, "visualize": {"avg_ms": 3.399, "count": 14, "max_ms": 3.687, "p95_ms": 3.687}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.256, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10437.75, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5643371d2000] error while decoding MB 76 10, bytestream 129 [h264 @ 0x56433723d140] cabac decode of qscale diff failed at 54 19 [h264 @ 0x56433723d140] error while decoding MB 54 19, bytestream 4796 [h264 @ 0x564337e9bb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564337e9bb00] error while decoding MB 0 44, bytestream 4036 [h264 @ 0x564337c2ef80] error while decoding MB 72 12, bytestream -5 [in#0/rtsp @ 0x5643371b7e00] Error during demuxing: Connection timed out [", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 357, "h264_decode_error_count": 1161, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:37:39+09:00", "last_error_summary": "[h264 @ 0x5643371d2000] error while decoding MB 76 10, bytestream 129 [h264 @ 0x56433723d140] cabac decode of qscale diff failed at 54 19 [h264 @ 0x56433723d140] error while decoding MB 54 19, bytestream 4796 [h264 @ 0x564337e9bb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564337e9bb00] error while decoding MB 0 44, bytestream 4036 [h264 @ 0x564337c2ef80] error while decoding MB 72 12, bytestream -5 [in#0/rtsp @ 0x5643371b7e00] Error during demuxing: Connection timed out [", "last_exit_at": "2026-07-01T01:27:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 631.887, "last_started_at": "2026-07-01T01:27:07+09:00", "pid": 71075, "process": {"exists": false, "pid": 71075}, "restart_count": 390, "restart_count_per_hour": 7, "rtsp_timeout_count": 332, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 391}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.612, "sample_count": 2698, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1215315, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:37:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.084, "last_frame_pushed_at": "2026-07-01T01:37:39+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-01T01:37:39+09:00", "mjpeg": {"age_sec": 0.069, "exists": true, "mtime": "2026-07-01T01:37:39.238324+09:00", "mtime_epoch": 1782837459.2383244, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124130}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:37:39+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4943.953, "name": "celery", "pid": 267, "rss_mb": 4943.953, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 113, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.773, "pushed": 3.773, "read": 3.773}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.745, "count": 113, "max_ms": 29.296, "p95_ms": 26.237}, "gstreamer_push": {"avg_ms": 1.215, "count": 113, "max_ms": 1.428, "p95_ms": 1.37}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.704, "count": 113, "max_ms": 33.086, "p95_ms": 32.111}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.141, "count": 113, "max_ms": 54.573, "p95_ms": 52.028}, "yolo11_640_fire_11class": {"avg_ms": 45.077, "count": 113, "max_ms": 51.308, "p95_ms": 49.513}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 46.398, "count": 113, "max_ms": 52.996, "p95_ms": 51.239}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.31, "count": 113, "max_ms": 84.904, "p95_ms": 83.826}, "read_interval": {"avg_ms": 265.114, "count": 113, "max_ms": 19810.59, "p95_ms": 97.69}, "triton_wait": {"avg_ms": 46.837, "count": 113, "max_ms": 54.585, "p95_ms": 52.398}, "visualize": {"avg_ms": 3.281, "count": 113, "max_ms": 4.212, "p95_ms": 3.69}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10437.352, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:37:39+09:00", "last_error_summary": "[h264 @ 0x5643371d2000] error while decoding MB 76 10, bytestream 129 [h264 @ 0x56433723d140] cabac decode of qscale diff failed at 54 19 [h264 @ 0x56433723d140] error while decoding MB 54 19, bytestream 4796 [h264 @ 0x564337e9bb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564337e9bb00] error while decoding MB 0 44, bytestream 4036 [h264 @ 0x564337c2ef80] error while decoding MB 72 12, bytestream -5 [in#0/rtsp @ 0x5643371b7e00] Error during demuxing: Connection timed out [", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 357, "h264_decode_error_count": 1161, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:37:39+09:00", "last_error_summary": "[h264 @ 0x5643371d2000] error while decoding MB 76 10, bytestream 129 [h264 @ 0x56433723d140] cabac decode of qscale diff failed at 54 19 [h264 @ 0x56433723d140] error while decoding MB 54 19, bytestream 4796 [h264 @ 0x564337e9bb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564337e9bb00] error while decoding MB 0 44, bytestream 4036 [h264 @ 0x564337c2ef80] error while decoding MB 72 12, bytestream -5 [in#0/rtsp @ 0x5643371b7e00] Error during demuxing: Connection timed out [", "last_exit_at": "2026-07-01T01:37:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 631.911, "last_started_at": "2026-07-01T01:27:07+09:00", "pid": 71075, "process": {"exists": false, "pid": 71075}, "restart_count": 390, "restart_count_per_hour": 7, "rtsp_timeout_count": 332, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 391}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.612, "sample_count": 2699, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1215316, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:37:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T01:37:39+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-01T01:37:39+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-01T01:37:39.238324+09:00", "mtime_epoch": 1782837459.2383244, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124130}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:37:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4943.953, "name": "celery", "pid": 267, "rss_mb": 4943.953, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.803, "pushed": 3.803, "read": 3.77}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.763, "count": 114, "max_ms": 29.296, "p95_ms": 26.237}, "gstreamer_push": {"avg_ms": 1.216, "count": 114, "max_ms": 1.428, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.698, "count": 114, "max_ms": 33.086, "p95_ms": 32.111}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.138, "count": 114, "max_ms": 54.573, "p95_ms": 52.028}, "yolo11_640_fire_11class": {"avg_ms": 45.082, "count": 114, "max_ms": 51.308, "p95_ms": 49.513}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 46.402, "count": 114, "max_ms": 52.996, "p95_ms": 51.239}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.378, "count": 114, "max_ms": 84.904, "p95_ms": 83.899}, "read_interval": {"avg_ms": 265.114, "count": 113, "max_ms": 19810.59, "p95_ms": 97.69}, "triton_wait": {"avg_ms": 46.837, "count": 114, "max_ms": 54.585, "p95_ms": 52.398}, "visualize": {"avg_ms": 3.316, "count": 114, "max_ms": 7.264, "p95_ms": 3.787}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10437.473, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 357, "h264_decode_error_count": 1161, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:37:39+09:00", "last_error_summary": "[h264 @ 0x5643371d2000] error while decoding MB 76 10, bytestream 129 [h264 @ 0x56433723d140] cabac decode of qscale diff failed at 54 19 [h264 @ 0x56433723d140] error while decoding MB 54 19, bytestream 4796 [h264 @ 0x564337e9bb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564337e9bb00] error while decoding MB 0 44, bytestream 4036 [h264 @ 0x564337c2ef80] error while decoding MB 72 12, bytestream -5 [in#0/rtsp @ 0x5643371b7e00] Error during demuxing: Connection timed out [", "last_exit_at": "2026-07-01T01:37:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 631.919, "last_started_at": "2026-07-01T01:27:07+09:00", "pid": 71075, "process": {"exists": false, "pid": 71075}, "restart_count": 390, "restart_count_per_hour": 7, "rtsp_timeout_count": 332, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 391}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.612, "sample_count": 2699, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1215316, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:37:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-01T01:37:39+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T01:37:39+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-01T01:37:39.238324+09:00", "mtime_epoch": 1782837459.2383244, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124130}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:37:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4943.953, "name": "celery", "pid": 267, "rss_mb": 4943.953, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.802, "pushed": 3.802, "read": 3.769}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.763, "count": 114, "max_ms": 29.296, "p95_ms": 26.237}, "gstreamer_push": {"avg_ms": 1.216, "count": 114, "max_ms": 1.428, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.698, "count": 114, "max_ms": 33.086, "p95_ms": 32.111}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.138, "count": 114, "max_ms": 54.573, "p95_ms": 52.028}, "yolo11_640_fire_11class": {"avg_ms": 45.082, "count": 114, "max_ms": 51.308, "p95_ms": 49.513}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 46.402, "count": 114, "max_ms": 52.996, "p95_ms": 51.239}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.378, "count": 114, "max_ms": 84.904, "p95_ms": 83.899}, "read_interval": {"avg_ms": 265.114, "count": 113, "max_ms": 19810.59, "p95_ms": 97.69}, "triton_wait": {"avg_ms": 46.837, "count": 114, "max_ms": 54.585, "p95_ms": 52.398}, "visualize": {"avg_ms": 3.316, "count": 114, "max_ms": 7.264, "p95_ms": 3.787}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10545.535, "limit_mb": 8796093022207.996}, "detail": {"pid": 71638, "restart_count": 391, "start_count": 392}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 357, "h264_decode_error_count": 1161, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:37:39+09:00", "last_error_summary": "[h264 @ 0x5643371d2000] error while decoding MB 76 10, bytestream 129 [h264 @ 0x56433723d140] cabac decode of qscale diff failed at 54 19 [h264 @ 0x56433723d140] error while decoding MB 54 19, bytestream 4796 [h264 @ 0x564337e9bb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564337e9bb00] error while decoding MB 0 44, bytestream 4036 [h264 @ 0x564337c2ef80] error while decoding MB 72 12, bytestream -5 [in#0/rtsp @ 0x5643371b7e00] Error during demuxing: Connection timed out [", "last_exit_at": "2026-07-01T01:37:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:37:41+09:00", "pid": 71638, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.047, "name": "ffmpeg", "pid": 71638, "rss_mb": 139.047, "state": "S (sleeping)", "threads": 106}, "restart_count": 391, "restart_count_per_hour": 8, "rtsp_timeout_count": 332, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 392}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.587, "sample_count": 2660, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1215316, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:37:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.464, "last_frame_pushed_at": "2026-07-01T01:37:39+09:00", "last_frame_read_age_sec": 4.55, "last_frame_read_at": "2026-07-01T01:37:39+09:00", "mjpeg": {"age_sec": 4.447, "exists": true, "mtime": "2026-07-01T01:37:39.338326+09:00", "mtime_epoch": 1782837459.3383262, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123778}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:37:43+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4943.953, "name": "celery", "pid": 267, "rss_mb": 4943.953, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.103, "pushed": 2.103, "read": 2.07}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.955, "count": 63, "max_ms": 28.655, "p95_ms": 25.054}, "gstreamer_push": {"avg_ms": 1.246, "count": 63, "max_ms": 1.388, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.194, "count": 63, "max_ms": 33.001, "p95_ms": 31.586}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.513, "count": 63, "max_ms": 54.573, "p95_ms": 52.464}, "yolo11_640_fire_11class": {"avg_ms": 46.139, "count": 63, "max_ms": 51.308, "p95_ms": 50.444}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.391, "count": 63, "max_ms": 52.996, "p95_ms": 52.205}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.144, "count": 63, "max_ms": 84.904, "p95_ms": 84.039}, "read_interval": {"avg_ms": 410.994, "count": 62, "max_ms": 19810.59, "p95_ms": 98.254}, "triton_wait": {"avg_ms": 48.139, "count": 63, "max_ms": 54.585, "p95_ms": 53.007}, "visualize": {"avg_ms": 3.459, "count": 63, "max_ms": 7.264, "p95_ms": 3.69}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11322}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10385.773, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562cd07c7400] error while decoding MB 39 21, bytestream -13 [in#0/rtsp @ 0x562ccffcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562ccffcde00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 157, "h264_decode_error_count": 162, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:40:31+09:00", "last_error_summary": "[h264 @ 0x562cd07c7400] error while decoding MB 39 21, bytestream -13 [in#0/rtsp @ 0x562ccffcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562ccffcde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:34:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 388.824, "last_started_at": "2026-07-01T01:34:02+09:00", "pid": 71486, "process": {"exists": false, "pid": 71486}, "restart_count": 156, "restart_count_per_hour": 11, "rtsp_timeout_count": 254, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 157}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.01, "sample_count": 2556, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1358691, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:40:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-01T01:40:31+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-01T01:40:31+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-01T01:40:31.589472+09:00", "mtime_epoch": 1782837631.5894723, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99981}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:40:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1701.48, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 54, "pushed_frames": 54, "read_frames": 54, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.806, "pushed": 1.806, "read": 1.806}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.523, "count": 54, "max_ms": 20.397, "p95_ms": 19.979}, "gstreamer_push": {"avg_ms": 1.246, "count": 54, "max_ms": 1.35, "p95_ms": 1.322}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.266, "count": 54, "max_ms": 60.339, "p95_ms": 58.739}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.655, "count": 54, "max_ms": 68.744, "p95_ms": 68.194}, "yolo11_640_fire_11class": {"avg_ms": 64.78, "count": 54, "max_ms": 68.207, "p95_ms": 67.363}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.727, "count": 54, "max_ms": 79.285, "p95_ms": 77.504}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.52, "count": 54, "max_ms": 105.504, "p95_ms": 105.069}, "read_interval": {"avg_ms": 554.471, "count": 54, "max_ms": 14914.293, "p95_ms": 120.515}, "triton_wait": {"avg_ms": 74.739, "count": 54, "max_ms": 79.297, "p95_ms": 77.516}, "visualize": {"avg_ms": 3.433, "count": 54, "max_ms": 3.952, "p95_ms": 3.589}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11322}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10385.391, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:40:31+09:00", "last_error_summary": "[h264 @ 0x562cd07c7400] error while decoding MB 39 21, bytestream -13 [in#0/rtsp @ 0x562ccffcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562ccffcde00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 157, "h264_decode_error_count": 162, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:40:31+09:00", "last_error_summary": "[h264 @ 0x562cd07c7400] error while decoding MB 39 21, bytestream -13 [in#0/rtsp @ 0x562ccffcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562ccffcde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:40:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 388.872, "last_started_at": "2026-07-01T01:34:02+09:00", "pid": 71486, "process": {"exists": false, "pid": 71486}, "restart_count": 156, "restart_count_per_hour": 11, "rtsp_timeout_count": 254, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 157}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.01, "sample_count": 2557, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1358692, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:40:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T01:40:31+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T01:40:31+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-01T01:40:31.589472+09:00", "mtime_epoch": 1782837631.5894723, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99981}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:40:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1701.48, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.836, "pushed": 1.836, "read": 1.803}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.521, "count": 55, "max_ms": 20.397, "p95_ms": 19.979}, "gstreamer_push": {"avg_ms": 1.247, "count": 55, "max_ms": 1.35, "p95_ms": 1.322}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.211, "count": 55, "max_ms": 60.339, "p95_ms": 58.739}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.688, "count": 55, "max_ms": 68.744, "p95_ms": 68.194}, "yolo11_640_fire_11class": {"avg_ms": 64.807, "count": 55, "max_ms": 68.207, "p95_ms": 67.363}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.749, "count": 55, "max_ms": 79.285, "p95_ms": 77.504}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.542, "count": 55, "max_ms": 105.504, "p95_ms": 105.069}, "read_interval": {"avg_ms": 554.471, "count": 54, "max_ms": 14914.293, "p95_ms": 120.515}, "triton_wait": {"avg_ms": 74.761, "count": 55, "max_ms": 79.297, "p95_ms": 77.516}, "visualize": {"avg_ms": 3.433, "count": 55, "max_ms": 3.952, "p95_ms": 3.589}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11322}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10385.406, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 157, "h264_decode_error_count": 162, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:40:31+09:00", "last_error_summary": "[h264 @ 0x562cd07c7400] error while decoding MB 39 21, bytestream -13 [in#0/rtsp @ 0x562ccffcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562ccffcde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:40:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 388.878, "last_started_at": "2026-07-01T01:34:02+09:00", "pid": 71486, "process": {"exists": false, "pid": 71486}, "restart_count": 156, "restart_count_per_hour": 11, "rtsp_timeout_count": 254, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 157}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.01, "sample_count": 2557, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1358692, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:40:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T01:40:31+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-01T01:40:31+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-01T01:40:31.589472+09:00", "mtime_epoch": 1782837631.5894723, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99981}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:40:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1701.48, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.836, "pushed": 1.836, "read": 1.803}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.521, "count": 55, "max_ms": 20.397, "p95_ms": 19.979}, "gstreamer_push": {"avg_ms": 1.247, "count": 55, "max_ms": 1.35, "p95_ms": 1.322}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.211, "count": 55, "max_ms": 60.339, "p95_ms": 58.739}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.688, "count": 55, "max_ms": 68.744, "p95_ms": 68.194}, "yolo11_640_fire_11class": {"avg_ms": 64.807, "count": 55, "max_ms": 68.207, "p95_ms": 67.363}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.749, "count": 55, "max_ms": 79.285, "p95_ms": 77.504}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.542, "count": 55, "max_ms": 105.504, "p95_ms": 105.069}, "read_interval": {"avg_ms": 554.471, "count": 54, "max_ms": 14914.293, "p95_ms": 120.515}, "triton_wait": {"avg_ms": 74.761, "count": 55, "max_ms": 79.297, "p95_ms": 77.516}, "visualize": {"avg_ms": 3.433, "count": 55, "max_ms": 3.952, "p95_ms": 3.589}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11322}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10559.625, "limit_mb": 8796093022207.996}, "detail": {"pid": 71745, "restart_count": 157, "start_count": 158}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 157, "h264_decode_error_count": 162, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:40:31+09:00", "last_error_summary": "[h264 @ 0x562cd07c7400] error while decoding MB 39 21, bytestream -13 [in#0/rtsp @ 0x562ccffcde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562ccffcde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:40:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:40:34+09:00", "pid": 71745, "process": {"exists": true, "fd_count": 5, "hwm_mb": 203.859, "name": "ffmpeg", "pid": 71745, "rss_mb": 203.859, "state": "S (sleeping)", "threads": 106}, "restart_count": 157, "restart_count_per_hour": 12, "rtsp_timeout_count": 254, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 158}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.1, "sample_count": 2517, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1358692, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:40:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.608, "last_frame_pushed_at": "2026-07-01T01:40:31+09:00", "last_frame_read_age_sec": 4.713, "last_frame_read_at": "2026-07-01T01:40:31+09:00", "mjpeg": {"age_sec": 4.595, "exists": true, "mtime": "2026-07-01T01:40:31.705474+09:00", "mtime_epoch": 1782837631.7054744, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95153}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:40:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1701.48, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 16, "pushed_frames": 16, "read_frames": 15, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.534, "pushed": 0.534, "read": 0.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.439, "count": 16, "max_ms": 18.93, "p95_ms": 18.93}, "gstreamer_push": {"avg_ms": 1.274, "count": 16, "max_ms": 1.329, "p95_ms": 1.329}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.156, "count": 16, "max_ms": 60.339, "p95_ms": 60.339}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.763, "count": 16, "max_ms": 68.744, "p95_ms": 68.744}, "yolo11_640_fire_11class": {"avg_ms": 63.41, "count": 16, "max_ms": 68.207, "p95_ms": 68.207}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.429, "count": 16, "max_ms": 78.903, "p95_ms": 78.903}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.173, "count": 16, "max_ms": 105.504, "p95_ms": 105.504}, "read_interval": {"avg_ms": 1690.177, "count": 15, "max_ms": 14914.293, "p95_ms": 14914.293}, "triton_wait": {"avg_ms": 72.441, "count": 16, "max_ms": 78.916, "p95_ms": 78.916}, "visualize": {"avg_ms": 3.444, "count": 16, "max_ms": 3.625, "p95_ms": 3.625}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10437.75, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556d08c10940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08c10940] error while decoding MB 0 15, bytestream 3423 [h264 @ 0x556d08cdf7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08cdf7c0] error while decoding MB 0 23, bytestream 5309 [h264 @ 0x556d08b41c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08b41c40] error while decoding MB 0 8, bytestream 7018 [h264 @ 0x556d0821c880] left block unavailable for reques", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 358, "h264_decode_error_count": 1166, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:42:01+09:00", "last_error_summary": "[h264 @ 0x556d08c10940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08c10940] error while decoding MB 0 15, bytestream 3423 [h264 @ 0x556d08cdf7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08cdf7c0] error while decoding MB 0 23, bytestream 5309 [h264 @ 0x556d08b41c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08b41c40] error while decoding MB 0 8, bytestream 7018 [h264 @ 0x556d0821c880] left block unavailable for reques", "last_exit_at": "2026-07-01T01:37:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 259.938, "last_started_at": "2026-07-01T01:37:41+09:00", "pid": 71638, "process": {"exists": false, "pid": 71638}, "restart_count": 391, "restart_count_per_hour": 8, "rtsp_timeout_count": 334, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 392}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.834, "sample_count": 2445, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1217564, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:42:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-01T01:42:01+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-01T01:42:01+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T01:42:01.659112+09:00", "mtime_epoch": 1782837721.6591117, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118420}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:42:01+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4945.242, "name": "celery", "pid": 267, "rss_mb": 4945.242, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 106, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.541, "pushed": 3.541, "read": 3.541}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.048, "count": 106, "max_ms": 29.811, "p95_ms": 25.405}, "gstreamer_push": {"avg_ms": 1.269, "count": 106, "max_ms": 1.758, "p95_ms": 1.536}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.229, "count": 106, "max_ms": 46.707, "p95_ms": 40.006}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.455, "count": 106, "max_ms": 66.649, "p95_ms": 64.487}, "yolo11_640_fire_11class": {"avg_ms": 52.399, "count": 106, "max_ms": 65.528, "p95_ms": 63.397}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.584, "count": 106, "max_ms": 75.376, "p95_ms": 72.451}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.596, "count": 106, "max_ms": 104.203, "p95_ms": 100.789}, "read_interval": {"avg_ms": 282.453, "count": 106, "max_ms": 10127.373, "p95_ms": 120.101}, "triton_wait": {"avg_ms": 57.66, "count": 106, "max_ms": 75.388, "p95_ms": 72.463}, "visualize": {"avg_ms": 3.302, "count": 106, "max_ms": 4.37, "p95_ms": 3.526}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10437.195, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:42:01+09:00", "last_error_summary": "[h264 @ 0x556d08c10940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08c10940] error while decoding MB 0 15, bytestream 3423 [h264 @ 0x556d08cdf7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08cdf7c0] error while decoding MB 0 23, bytestream 5309 [h264 @ 0x556d08b41c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08b41c40] error while decoding MB 0 8, bytestream 7018 [h264 @ 0x556d0821c880] left block unavailable for reques", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 358, "h264_decode_error_count": 1166, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:42:01+09:00", "last_error_summary": "[h264 @ 0x556d08c10940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08c10940] error while decoding MB 0 15, bytestream 3423 [h264 @ 0x556d08cdf7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08cdf7c0] error while decoding MB 0 23, bytestream 5309 [h264 @ 0x556d08b41c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08b41c40] error while decoding MB 0 8, bytestream 7018 [h264 @ 0x556d0821c880] left block unavailable for reques", "last_exit_at": "2026-07-01T01:42:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 259.967, "last_started_at": "2026-07-01T01:37:41+09:00", "pid": 71638, "process": {"exists": false, "pid": 71638}, "restart_count": 391, "restart_count_per_hour": 8, "rtsp_timeout_count": 334, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 392}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.834, "sample_count": 2446, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1217565, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:42:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T01:42:01+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-01T01:42:01+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-01T01:42:01.659112+09:00", "mtime_epoch": 1782837721.6591117, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118420}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:42:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4945.242, "name": "celery", "pid": 267, "rss_mb": 4945.242, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.571, "pushed": 3.571, "read": 3.538}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.097, "count": 107, "max_ms": 29.811, "p95_ms": 26.372}, "gstreamer_push": {"avg_ms": 1.272, "count": 107, "max_ms": 1.758, "p95_ms": 1.536}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.212, "count": 107, "max_ms": 46.707, "p95_ms": 40.006}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.382, "count": 107, "max_ms": 66.649, "p95_ms": 64.487}, "yolo11_640_fire_11class": {"avg_ms": 52.326, "count": 107, "max_ms": 65.528, "p95_ms": 63.397}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.48, "count": 107, "max_ms": 75.376, "p95_ms": 72.451}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.574, "count": 107, "max_ms": 104.203, "p95_ms": 100.789}, "read_interval": {"avg_ms": 282.453, "count": 106, "max_ms": 10127.373, "p95_ms": 120.101}, "triton_wait": {"avg_ms": 57.555, "count": 107, "max_ms": 75.388, "p95_ms": 72.463}, "visualize": {"avg_ms": 3.33, "count": 107, "max_ms": 6.249, "p95_ms": 3.527}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10437.051, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 358, "h264_decode_error_count": 1166, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:42:01+09:00", "last_error_summary": "[h264 @ 0x556d08c10940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08c10940] error while decoding MB 0 15, bytestream 3423 [h264 @ 0x556d08cdf7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08cdf7c0] error while decoding MB 0 23, bytestream 5309 [h264 @ 0x556d08b41c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08b41c40] error while decoding MB 0 8, bytestream 7018 [h264 @ 0x556d0821c880] left block unavailable for reques", "last_exit_at": "2026-07-01T01:42:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 259.979, "last_started_at": "2026-07-01T01:37:41+09:00", "pid": 71638, "process": {"exists": false, "pid": 71638}, "restart_count": 391, "restart_count_per_hour": 8, "rtsp_timeout_count": 334, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 392}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.834, "sample_count": 2445, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1217565, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:42:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.021, "last_frame_pushed_at": "2026-07-01T01:42:01+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-01T01:42:01+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-01T01:42:01.659112+09:00", "mtime_epoch": 1782837721.6591117, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118420}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:42:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4945.242, "name": "celery", "pid": 267, "rss_mb": 4945.242, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.57, "pushed": 3.57, "read": 3.536}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.097, "count": 107, "max_ms": 29.811, "p95_ms": 26.372}, "gstreamer_push": {"avg_ms": 1.272, "count": 107, "max_ms": 1.758, "p95_ms": 1.536}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.212, "count": 107, "max_ms": 46.707, "p95_ms": 40.006}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.382, "count": 107, "max_ms": 66.649, "p95_ms": 64.487}, "yolo11_640_fire_11class": {"avg_ms": 52.326, "count": 107, "max_ms": 65.528, "p95_ms": 63.397}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.48, "count": 107, "max_ms": 75.376, "p95_ms": 72.451}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.574, "count": 107, "max_ms": 104.203, "p95_ms": 100.789}, "read_interval": {"avg_ms": 282.453, "count": 106, "max_ms": 10127.373, "p95_ms": 120.101}, "triton_wait": {"avg_ms": 57.555, "count": 107, "max_ms": 75.388, "p95_ms": 72.463}, "visualize": {"avg_ms": 3.33, "count": 107, "max_ms": 6.249, "p95_ms": 3.527}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10544.871, "limit_mb": 8796093022207.996}, "detail": {"pid": 71897, "restart_count": 392, "start_count": 393}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 358, "h264_decode_error_count": 1166, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:42:01+09:00", "last_error_summary": "[h264 @ 0x556d08c10940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08c10940] error while decoding MB 0 15, bytestream 3423 [h264 @ 0x556d08cdf7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08cdf7c0] error while decoding MB 0 23, bytestream 5309 [h264 @ 0x556d08b41c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556d08b41c40] error while decoding MB 0 8, bytestream 7018 [h264 @ 0x556d0821c880] left block unavailable for reques", "last_exit_at": "2026-07-01T01:42:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:42:04+09:00", "pid": 71897, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.758, "name": "ffmpeg", "pid": 71897, "rss_mb": 138.758, "state": "S (sleeping)", "threads": 106}, "restart_count": 392, "restart_count_per_hour": 9, "rtsp_timeout_count": 334, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 393}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.858, "sample_count": 2395, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1217565, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:42:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.631, "last_frame_pushed_at": "2026-07-01T01:42:01+09:00", "last_frame_read_age_sec": 4.717, "last_frame_read_at": "2026-07-01T01:42:01+09:00", "mjpeg": {"age_sec": 4.606, "exists": true, "mtime": "2026-07-01T01:42:01.767114+09:00", "mtime_epoch": 1782837721.7671137, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118473}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:42:06+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4945.242, "name": "celery", "pid": 267, "rss_mb": 4945.242, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.97, "pushed": 1.97, "read": 1.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.56, "count": 59, "max_ms": 29.811, "p95_ms": 23.306}, "gstreamer_push": {"avg_ms": 1.255, "count": 59, "max_ms": 1.527, "p95_ms": 1.358}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.167, "count": 59, "max_ms": 46.707, "p95_ms": 40.871}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.133, "count": 59, "max_ms": 66.649, "p95_ms": 65.222}, "yolo11_640_fire_11class": {"avg_ms": 56.869, "count": 59, "max_ms": 65.528, "p95_ms": 64.308}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.037, "count": 59, "max_ms": 75.376, "p95_ms": 73.888}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.733, "count": 59, "max_ms": 104.203, "p95_ms": 101.536}, "read_interval": {"avg_ms": 436.899, "count": 58, "max_ms": 10127.373, "p95_ms": 125.424}, "triton_wait": {"avg_ms": 64.105, "count": 59, "max_ms": 75.388, "p95_ms": 73.901}, "visualize": {"avg_ms": 3.421, "count": 59, "max_ms": 6.249, "p95_ms": 3.527}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11336}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10380.629, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563f6e1be040] error while decoding MB 0 27, bytestream -7 [in#0/rtsp @ 0x563f6d827e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f6d827e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 158, "h264_decode_error_count": 163, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:44:03+09:00", "last_error_summary": "[h264 @ 0x563f6e1be040] error while decoding MB 0 27, bytestream -7 [in#0/rtsp @ 0x563f6d827e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f6d827e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:40:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.517, "last_started_at": "2026-07-01T01:40:34+09:00", "pid": 71745, "process": {"exists": false, "pid": 71745}, "restart_count": 157, "restart_count_per_hour": 11, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 158}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.089, "sample_count": 2191, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1360329, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:44:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-01T01:44:03+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-01T01:44:03+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T01:44:03.757329+09:00", "mtime_epoch": 1782837843.757329, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:44:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1702.254, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.683, "pushed": 0.683, "read": 0.74}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.176, "count": 12, "max_ms": 19.106, "p95_ms": 19.106}, "gstreamer_push": {"avg_ms": 1.218, "count": 12, "max_ms": 1.344, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.201, "count": 12, "max_ms": 62.328, "p95_ms": 62.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.279, "count": 12, "max_ms": 65.926, "p95_ms": 65.926}, "yolo11_640_fire_11class": {"avg_ms": 58.468, "count": 12, "max_ms": 64.965, "p95_ms": 64.965}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.706, "count": 12, "max_ms": 74.132, "p95_ms": 74.132}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.215, "count": 12, "max_ms": 100.369, "p95_ms": 100.369}, "read_interval": {"avg_ms": 2615.918, "count": 13, "max_ms": 16500.182, "p95_ms": 16500.182}, "triton_wait": {"avg_ms": 66.718, "count": 12, "max_ms": 74.145, "p95_ms": 74.145}, "visualize": {"avg_ms": 3.507, "count": 12, "max_ms": 3.884, "p95_ms": 3.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.568, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11336}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10380.109, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:44:03+09:00", "last_error_summary": "[h264 @ 0x563f6e1be040] error while decoding MB 0 27, bytestream -7 [in#0/rtsp @ 0x563f6d827e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f6d827e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 158, "h264_decode_error_count": 163, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:44:03+09:00", "last_error_summary": "[h264 @ 0x563f6e1be040] error while decoding MB 0 27, bytestream -7 [in#0/rtsp @ 0x563f6d827e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f6d827e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:44:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.568, "last_started_at": "2026-07-01T01:40:34+09:00", "pid": 71745, "process": {"exists": false, "pid": 71745}, "restart_count": 157, "restart_count_per_hour": 11, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 158}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.089, "sample_count": 2191, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1360330, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:44:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T01:44:03+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-01T01:44:03+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-01T01:44:03.757329+09:00", "mtime_epoch": 1782837843.757329, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:44:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1702.254, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.738, "pushed": 0.738, "read": 0.738}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.178, "count": 13, "max_ms": 19.106, "p95_ms": 19.106}, "gstreamer_push": {"avg_ms": 1.218, "count": 13, "max_ms": 1.344, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.473, "count": 13, "max_ms": 62.328, "p95_ms": 62.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.953, "count": 13, "max_ms": 68.037, "p95_ms": 68.037}, "yolo11_640_fire_11class": {"avg_ms": 59.203, "count": 13, "max_ms": 68.024, "p95_ms": 68.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.544, "count": 13, "max_ms": 77.596, "p95_ms": 77.596}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.046, "count": 13, "max_ms": 104.015, "p95_ms": 104.015}, "read_interval": {"avg_ms": 2615.918, "count": 13, "max_ms": 16500.182, "p95_ms": 16500.182}, "triton_wait": {"avg_ms": 67.556, "count": 13, "max_ms": 77.608, "p95_ms": 77.608}, "visualize": {"avg_ms": 3.504, "count": 13, "max_ms": 3.884, "p95_ms": 3.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.618, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11336}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10380.066, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 158, "h264_decode_error_count": 163, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:44:03+09:00", "last_error_summary": "[h264 @ 0x563f6e1be040] error while decoding MB 0 27, bytestream -7 [in#0/rtsp @ 0x563f6d827e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f6d827e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:44:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.574, "last_started_at": "2026-07-01T01:40:34+09:00", "pid": 71745, "process": {"exists": false, "pid": 71745}, "restart_count": 157, "restart_count_per_hour": 11, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 158}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.089, "sample_count": 2191, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1360330, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:44:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T01:44:03+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-01T01:44:03+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-01T01:44:03.757329+09:00", "mtime_epoch": 1782837843.757329, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:44:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1702.254, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.738, "pushed": 0.738, "read": 0.738}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.178, "count": 13, "max_ms": 19.106, "p95_ms": 19.106}, "gstreamer_push": {"avg_ms": 1.218, "count": 13, "max_ms": 1.344, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.473, "count": 13, "max_ms": 62.328, "p95_ms": 62.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.953, "count": 13, "max_ms": 68.037, "p95_ms": 68.037}, "yolo11_640_fire_11class": {"avg_ms": 59.203, "count": 13, "max_ms": 68.024, "p95_ms": 68.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.544, "count": 13, "max_ms": 77.596, "p95_ms": 77.596}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.046, "count": 13, "max_ms": 104.015, "p95_ms": 104.015}, "read_interval": {"avg_ms": 2615.918, "count": 13, "max_ms": 16500.182, "p95_ms": 16500.182}, "triton_wait": {"avg_ms": 67.556, "count": 13, "max_ms": 77.608, "p95_ms": 77.608}, "visualize": {"avg_ms": 3.504, "count": 13, "max_ms": 3.884, "p95_ms": 3.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.624, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11336}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10551.836, "limit_mb": 8796093022207.996}, "detail": {"pid": 72004, "restart_count": 158, "start_count": 159}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 158, "h264_decode_error_count": 163, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:44:03+09:00", "last_error_summary": "[h264 @ 0x563f6e1be040] error while decoding MB 0 27, bytestream -7 [in#0/rtsp @ 0x563f6d827e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f6d827e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:44:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:44:06+09:00", "pid": 72004, "process": {"exists": true, "fd_count": 5, "hwm_mb": 201.094, "name": "ffmpeg", "pid": 72004, "rss_mb": 201.094, "state": "S (sleeping)", "threads": 106}, "restart_count": 158, "restart_count_per_hour": 12, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 159}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.06, "sample_count": 2154, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1360330, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:44:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.382, "last_frame_pushed_at": "2026-07-01T01:44:03+09:00", "last_frame_read_age_sec": 4.488, "last_frame_read_at": "2026-07-01T01:44:03+09:00", "mjpeg": {"age_sec": 4.367, "exists": true, "mtime": "2026-07-01T01:44:03.877331+09:00", "mtime_epoch": 1782837843.8773313, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97087}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:44:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1702.254, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.591, "pushed": 0.591, "read": 0.591}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.178, "count": 13, "max_ms": 19.106, "p95_ms": 19.106}, "gstreamer_push": {"avg_ms": 1.218, "count": 13, "max_ms": 1.344, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.473, "count": 13, "max_ms": 62.328, "p95_ms": 62.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.953, "count": 13, "max_ms": 68.037, "p95_ms": 68.037}, "yolo11_640_fire_11class": {"avg_ms": 59.203, "count": 13, "max_ms": 68.024, "p95_ms": 68.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.544, "count": 13, "max_ms": 77.596, "p95_ms": 77.596}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.046, "count": 13, "max_ms": 104.015, "p95_ms": 104.015}, "read_interval": {"avg_ms": 2615.918, "count": 13, "max_ms": 16500.182, "p95_ms": 16500.182}, "triton_wait": {"avg_ms": 67.556, "count": 13, "max_ms": 77.608, "p95_ms": 77.608}, "visualize": {"avg_ms": 3.504, "count": 13, "max_ms": 3.884, "p95_ms": 3.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.995, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10440.441, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ad4c9cd1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ad4c9cd1c0] error while decoding MB 0 22, bytestream 6856 [h264 @ 0x55ad4c8f0840] error while decoding MB 1 15, bytestream -10 [h264 @ 0x55ad4c919540] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4c919540] error while decoding MB 54 0, bytestream 8508 [h264 @ 0x55ad4d4c81c0] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4d4c81c0] error while decoding MB 9 0, bytestream 8", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 359, "h264_decode_error_count": 1170, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:47:00+09:00", "last_error_summary": "[h264 @ 0x55ad4c9cd1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ad4c9cd1c0] error while decoding MB 0 22, bytestream 6856 [h264 @ 0x55ad4c8f0840] error while decoding MB 1 15, bytestream -10 [h264 @ 0x55ad4c919540] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4c919540] error while decoding MB 54 0, bytestream 8508 [h264 @ 0x55ad4d4c81c0] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4d4c81c0] error while decoding MB 9 0, bytestream 8", "last_exit_at": "2026-07-01T01:42:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.04, "last_started_at": "2026-07-01T01:42:04+09:00", "pid": 71897, "process": {"exists": false, "pid": 71897}, "restart_count": 392, "restart_count_per_hour": 8, "rtsp_timeout_count": 336, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 393}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.888, "sample_count": 2485, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1220040, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:47:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.082, "last_frame_pushed_at": "2026-07-01T01:47:00+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T01:47:00+09:00", "mjpeg": {"age_sec": 0.068, "exists": true, "mtime": "2026-07-01T01:47:00.344528+09:00", "mtime_epoch": 1782838020.3445282, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118179}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:47:00+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4946.273, "name": "celery", "pid": 267, "rss_mb": 4946.273, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 61, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.0, "pushed": 2.0, "read": 2.034}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.374, "count": 60, "max_ms": 24.33, "p95_ms": 22.021}, "gstreamer_push": {"avg_ms": 1.229, "count": 60, "max_ms": 1.601, "p95_ms": 1.316}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.486, "count": 60, "max_ms": 42.245, "p95_ms": 41.614}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.824, "count": 60, "max_ms": 72.306, "p95_ms": 66.287}, "yolo11_640_fire_11class": {"avg_ms": 59.87, "count": 60, "max_ms": 71.091, "p95_ms": 65.204}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.294, "count": 60, "max_ms": 80.897, "p95_ms": 75.532}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.82, "count": 60, "max_ms": 108.618, "p95_ms": 103.901}, "read_interval": {"avg_ms": 492.503, "count": 61, "max_ms": 23338.073, "p95_ms": 119.003}, "triton_wait": {"avg_ms": 68.318, "count": 60, "max_ms": 80.909, "p95_ms": 75.564}, "visualize": {"avg_ms": 3.358, "count": 60, "max_ms": 3.591, "p95_ms": 3.572}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10440.391, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:47:00+09:00", "last_error_summary": "[h264 @ 0x55ad4c9cd1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ad4c9cd1c0] error while decoding MB 0 22, bytestream 6856 [h264 @ 0x55ad4c8f0840] error while decoding MB 1 15, bytestream -10 [h264 @ 0x55ad4c919540] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4c919540] error while decoding MB 54 0, bytestream 8508 [h264 @ 0x55ad4d4c81c0] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4d4c81c0] error while decoding MB 9 0, bytestream 8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 359, "h264_decode_error_count": 1170, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:47:00+09:00", "last_error_summary": "[h264 @ 0x55ad4c9cd1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ad4c9cd1c0] error while decoding MB 0 22, bytestream 6856 [h264 @ 0x55ad4c8f0840] error while decoding MB 1 15, bytestream -10 [h264 @ 0x55ad4c919540] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4c919540] error while decoding MB 54 0, bytestream 8508 [h264 @ 0x55ad4d4c81c0] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4d4c81c0] error while decoding MB 9 0, bytestream 8", "last_exit_at": "2026-07-01T01:47:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.064, "last_started_at": "2026-07-01T01:42:04+09:00", "pid": 71897, "process": {"exists": false, "pid": 71897}, "restart_count": 392, "restart_count_per_hour": 8, "rtsp_timeout_count": 336, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 393}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.888, "sample_count": 2486, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1220041, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:47:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T01:47:00+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-01T01:47:00+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-01T01:47:00.344528+09:00", "mtime_epoch": 1782838020.3445282, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118179}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:47:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4946.273, "name": "celery", "pid": 267, "rss_mb": 4946.273, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.039, "pushed": 2.039, "read": 2.005}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.491, "count": 61, "max_ms": 27.551, "p95_ms": 22.55}, "gstreamer_push": {"avg_ms": 1.231, "count": 61, "max_ms": 1.601, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.429, "count": 61, "max_ms": 42.245, "p95_ms": 41.614}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.562, "count": 61, "max_ms": 72.306, "p95_ms": 66.287}, "yolo11_640_fire_11class": {"avg_ms": 59.601, "count": 61, "max_ms": 71.091, "p95_ms": 65.204}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.918, "count": 61, "max_ms": 80.897, "p95_ms": 75.532}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.601, "count": 61, "max_ms": 108.618, "p95_ms": 103.901}, "read_interval": {"avg_ms": 498.802, "count": 60, "max_ms": 23338.073, "p95_ms": 119.003}, "triton_wait": {"avg_ms": 67.942, "count": 61, "max_ms": 80.909, "p95_ms": 75.564}, "visualize": {"avg_ms": 3.392, "count": 61, "max_ms": 5.422, "p95_ms": 3.575}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10440.105, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 359, "h264_decode_error_count": 1170, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:47:00+09:00", "last_error_summary": "[h264 @ 0x55ad4c9cd1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ad4c9cd1c0] error while decoding MB 0 22, bytestream 6856 [h264 @ 0x55ad4c8f0840] error while decoding MB 1 15, bytestream -10 [h264 @ 0x55ad4c919540] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4c919540] error while decoding MB 54 0, bytestream 8508 [h264 @ 0x55ad4d4c81c0] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4d4c81c0] error while decoding MB 9 0, bytestream 8", "last_exit_at": "2026-07-01T01:47:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.073, "last_started_at": "2026-07-01T01:42:04+09:00", "pid": 71897, "process": {"exists": false, "pid": 71897}, "restart_count": 392, "restart_count_per_hour": 8, "rtsp_timeout_count": 336, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 393}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.888, "sample_count": 2486, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1220041, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:47:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T01:47:00+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T01:47:00+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-01T01:47:00.344528+09:00", "mtime_epoch": 1782838020.3445282, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118179}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:47:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4946.273, "name": "celery", "pid": 267, "rss_mb": 4946.273, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.038, "pushed": 2.038, "read": 2.005}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.491, "count": 61, "max_ms": 27.551, "p95_ms": 22.55}, "gstreamer_push": {"avg_ms": 1.231, "count": 61, "max_ms": 1.601, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.429, "count": 61, "max_ms": 42.245, "p95_ms": 41.614}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.562, "count": 61, "max_ms": 72.306, "p95_ms": 66.287}, "yolo11_640_fire_11class": {"avg_ms": 59.601, "count": 61, "max_ms": 71.091, "p95_ms": 65.204}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.918, "count": 61, "max_ms": 80.897, "p95_ms": 75.532}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.601, "count": 61, "max_ms": 108.618, "p95_ms": 103.901}, "read_interval": {"avg_ms": 498.802, "count": 60, "max_ms": 23338.073, "p95_ms": 119.003}, "triton_wait": {"avg_ms": 67.942, "count": 61, "max_ms": 80.909, "p95_ms": 75.564}, "visualize": {"avg_ms": 3.392, "count": 61, "max_ms": 5.422, "p95_ms": 3.575}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10553.324, "limit_mb": 8796093022207.996}, "detail": {"pid": 72156, "restart_count": 393, "start_count": 394}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 359, "h264_decode_error_count": 1170, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:47:00+09:00", "last_error_summary": "[h264 @ 0x55ad4c9cd1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ad4c9cd1c0] error while decoding MB 0 22, bytestream 6856 [h264 @ 0x55ad4c8f0840] error while decoding MB 1 15, bytestream -10 [h264 @ 0x55ad4c919540] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4c919540] error while decoding MB 54 0, bytestream 8508 [h264 @ 0x55ad4d4c81c0] top block unavailable for requested intra mode -1 [h264 @ 0x55ad4d4c81c0] error while decoding MB 9 0, bytestream 8", "last_exit_at": "2026-07-01T01:47:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:47:03+09:00", "pid": 72156, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.711, "name": "ffmpeg", "pid": 72156, "rss_mb": 144.711, "state": "S (sleeping)", "threads": 106}, "restart_count": 393, "restart_count_per_hour": 9, "rtsp_timeout_count": 336, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 394}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.888, "sample_count": 2476, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1220041, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:47:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.657, "last_frame_pushed_at": "2026-07-01T01:47:00+09:00", "last_frame_read_age_sec": 4.743, "last_frame_read_at": "2026-07-01T01:47:00+09:00", "mjpeg": {"age_sec": 4.642, "exists": true, "mtime": "2026-07-01T01:47:00.444530+09:00", "mtime_epoch": 1782838020.44453, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:47:05+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4946.273, "name": "celery", "pid": 267, "rss_mb": 4946.273, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 20, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.701, "pushed": 0.701, "read": 0.668}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.879, "count": 21, "max_ms": 27.551, "p95_ms": 24.33}, "gstreamer_push": {"avg_ms": 1.25, "count": 21, "max_ms": 1.601, "p95_ms": 1.425}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.606, "count": 21, "max_ms": 40.021, "p95_ms": 35.025}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.81, "count": 21, "max_ms": 65.949, "p95_ms": 65.827}, "yolo11_640_fire_11class": {"avg_ms": 53.817, "count": 21, "max_ms": 65.025, "p95_ms": 64.757}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.645, "count": 21, "max_ms": 74.909, "p95_ms": 74.334}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.879, "count": 21, "max_ms": 103.507, "p95_ms": 103.139}, "read_interval": {"avg_ms": 1264.999, "count": 20, "max_ms": 23338.073, "p95_ms": 118.38}, "triton_wait": {"avg_ms": 59.69, "count": 21, "max_ms": 74.921, "p95_ms": 74.347}, "visualize": {"avg_ms": 3.434, "count": 21, "max_ms": 5.422, "p95_ms": 3.591}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10442.266, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 34, bytestream 3570 [h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 27, bytestream 1494 [h264 @ 0x55634d970380] error while decoding MB 41 25, bytestream -22 [h264 @ 0x55634d8d8580] error while decoding MB 74 26, bytestream -5 [h264 @ 0x55634e3b3e00] error while decoding MB 0 31, bytestream ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 360, "h264_decode_error_count": 1175, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:50:19+09:00", "last_error_summary": "[h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 34, bytestream 3570 [h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 27, bytestream 1494 [h264 @ 0x55634d970380] error while decoding MB 41 25, bytestream -22 [h264 @ 0x55634d8d8580] error while decoding MB 74 26, bytestream -5 [h264 @ 0x55634e3b3e00] error while decoding MB 0 31, bytestream ", "last_exit_at": "2026-07-01T01:47:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.606, "last_started_at": "2026-07-01T01:47:03+09:00", "pid": 72156, "process": {"exists": false, "pid": 72156}, "restart_count": 393, "restart_count_per_hour": 8, "rtsp_timeout_count": 338, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 394}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.336, "sample_count": 2231, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1221593, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:50:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T01:50:19+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T01:50:19+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-01T01:50:19.628174+09:00", "mtime_epoch": 1782838219.628174, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117756}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:50:19+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4947.047, "name": "celery", "pid": 267, "rss_mb": 4947.047, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.237, "pushed": 3.237, "read": 3.237}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.276, "count": 97, "max_ms": 23.206, "p95_ms": 22.08}, "gstreamer_push": {"avg_ms": 1.228, "count": 97, "max_ms": 1.346, "p95_ms": 1.319}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.443, "count": 97, "max_ms": 44.067, "p95_ms": 42.212}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.293, "count": 97, "max_ms": 67.827, "p95_ms": 66.752}, "yolo11_640_fire_11class": {"avg_ms": 61.179, "count": 97, "max_ms": 66.56, "p95_ms": 65.352}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.448, "count": 97, "max_ms": 76.167, "p95_ms": 75.125}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.808, "count": 97, "max_ms": 104.204, "p95_ms": 103.399}, "read_interval": {"avg_ms": 309.267, "count": 97, "max_ms": 19098.768, "p95_ms": 118.506}, "triton_wait": {"avg_ms": 70.46, "count": 97, "max_ms": 76.179, "p95_ms": 75.137}, "visualize": {"avg_ms": 3.388, "count": 97, "max_ms": 3.974, "p95_ms": 3.579}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10442.094, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:50:19+09:00", "last_error_summary": "[h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 34, bytestream 3570 [h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 27, bytestream 1494 [h264 @ 0x55634d970380] error while decoding MB 41 25, bytestream -22 [h264 @ 0x55634d8d8580] error while decoding MB 74 26, bytestream -5 [h264 @ 0x55634e3b3e00] error while decoding MB 0 31, bytestream ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 360, "h264_decode_error_count": 1175, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:50:19+09:00", "last_error_summary": "[h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 34, bytestream 3570 [h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 27, bytestream 1494 [h264 @ 0x55634d970380] error while decoding MB 41 25, bytestream -22 [h264 @ 0x55634d8d8580] error while decoding MB 74 26, bytestream -5 [h264 @ 0x55634e3b3e00] error while decoding MB 0 31, bytestream ", "last_exit_at": "2026-07-01T01:50:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.63, "last_started_at": "2026-07-01T01:47:03+09:00", "pid": 72156, "process": {"exists": false, "pid": 72156}, "restart_count": 393, "restart_count_per_hour": 8, "rtsp_timeout_count": 338, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 394}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.336, "sample_count": 2232, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1221594, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:50:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T01:50:19+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-01T01:50:19+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T01:50:19.628174+09:00", "mtime_epoch": 1782838219.628174, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117756}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:50:19+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4947.047, "name": "celery", "pid": 267, "rss_mb": 4947.047, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.268, "pushed": 3.268, "read": 3.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.334, "count": 98, "max_ms": 26.007, "p95_ms": 22.456}, "gstreamer_push": {"avg_ms": 1.229, "count": 98, "max_ms": 1.363, "p95_ms": 1.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.424, "count": 98, "max_ms": 44.067, "p95_ms": 42.212}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.112, "count": 98, "max_ms": 67.827, "p95_ms": 66.752}, "yolo11_640_fire_11class": {"avg_ms": 60.966, "count": 98, "max_ms": 66.56, "p95_ms": 65.352}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.193, "count": 98, "max_ms": 76.167, "p95_ms": 75.125}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.631, "count": 98, "max_ms": 104.204, "p95_ms": 103.399}, "read_interval": {"avg_ms": 309.267, "count": 97, "max_ms": 19098.768, "p95_ms": 118.506}, "triton_wait": {"avg_ms": 70.205, "count": 98, "max_ms": 76.179, "p95_ms": 75.137}, "visualize": {"avg_ms": 3.401, "count": 98, "max_ms": 4.632, "p95_ms": 3.602}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10442.082, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 360, "h264_decode_error_count": 1175, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:50:19+09:00", "last_error_summary": "[h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 34, bytestream 3570 [h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 27, bytestream 1494 [h264 @ 0x55634d970380] error while decoding MB 41 25, bytestream -22 [h264 @ 0x55634d8d8580] error while decoding MB 74 26, bytestream -5 [h264 @ 0x55634e3b3e00] error while decoding MB 0 31, bytestream ", "last_exit_at": "2026-07-01T01:50:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.638, "last_started_at": "2026-07-01T01:47:03+09:00", "pid": 72156, "process": {"exists": false, "pid": 72156}, "restart_count": 393, "restart_count_per_hour": 8, "rtsp_timeout_count": 338, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 394}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.336, "sample_count": 2232, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1221594, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:50:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T01:50:19+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-01T01:50:19+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-01T01:50:19.628174+09:00", "mtime_epoch": 1782838219.628174, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117756}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:50:19+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4947.047, "name": "celery", "pid": 267, "rss_mb": 4947.047, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.267, "pushed": 3.267, "read": 3.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.343, "count": 97, "max_ms": 26.007, "p95_ms": 22.456}, "gstreamer_push": {"avg_ms": 1.229, "count": 98, "max_ms": 1.363, "p95_ms": 1.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.355, "count": 97, "max_ms": 44.067, "p95_ms": 42.212}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.103, "count": 97, "max_ms": 67.827, "p95_ms": 66.752}, "yolo11_640_fire_11class": {"avg_ms": 60.957, "count": 97, "max_ms": 66.56, "p95_ms": 65.352}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.175, "count": 97, "max_ms": 76.167, "p95_ms": 75.125}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.631, "count": 98, "max_ms": 104.204, "p95_ms": 103.399}, "read_interval": {"avg_ms": 309.267, "count": 97, "max_ms": 19098.768, "p95_ms": 118.506}, "triton_wait": {"avg_ms": 70.187, "count": 97, "max_ms": 76.179, "p95_ms": 75.137}, "visualize": {"avg_ms": 3.402, "count": 97, "max_ms": 4.632, "p95_ms": 3.602}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10549.414, "limit_mb": 8796093022207.996}, "detail": {"pid": 72263, "restart_count": 394, "start_count": 395}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 360, "h264_decode_error_count": 1175, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:50:19+09:00", "last_error_summary": "[h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 34, bytestream 3570 [h264 @ 0x55634e551b00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55634e551b00] error while decoding MB 0 27, bytestream 1494 [h264 @ 0x55634d970380] error while decoding MB 41 25, bytestream -22 [h264 @ 0x55634d8d8580] error while decoding MB 74 26, bytestream -5 [h264 @ 0x55634e3b3e00] error while decoding MB 0 31, bytestream ", "last_exit_at": "2026-07-01T01:50:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:50:22+09:00", "pid": 72263, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.406, "name": "ffmpeg", "pid": 72263, "rss_mb": 138.406, "state": "S (sleeping)", "threads": 106}, "restart_count": 394, "restart_count_per_hour": 9, "rtsp_timeout_count": 338, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 395}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.366, "sample_count": 2186, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1221594, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:50:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.467, "last_frame_pushed_at": "2026-07-01T01:50:19+09:00", "last_frame_read_age_sec": 4.551, "last_frame_read_at": "2026-07-01T01:50:19+09:00", "mjpeg": {"age_sec": 4.453, "exists": true, "mtime": "2026-07-01T01:50:19.724176+09:00", "mtime_epoch": 1782838219.724176, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:50:24+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4947.047, "name": "celery", "pid": 267, "rss_mb": 4947.047, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.972, "pushed": 1.972, "read": 1.939}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.46, "count": 59, "max_ms": 26.007, "p95_ms": 22.729}, "gstreamer_push": {"avg_ms": 1.23, "count": 59, "max_ms": 1.363, "p95_ms": 1.341}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.384, "count": 59, "max_ms": 44.067, "p95_ms": 41.958}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.721, "count": 59, "max_ms": 67.26, "p95_ms": 66.345}, "yolo11_640_fire_11class": {"avg_ms": 59.406, "count": 59, "max_ms": 65.783, "p95_ms": 64.674}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.267, "count": 59, "max_ms": 75.234, "p95_ms": 74.64}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.841, "count": 59, "max_ms": 103.568, "p95_ms": 103.125}, "read_interval": {"avg_ms": 439.108, "count": 58, "max_ms": 19098.768, "p95_ms": 118.352}, "triton_wait": {"avg_ms": 68.279, "count": 59, "max_ms": 75.245, "p95_ms": 74.653}, "visualize": {"avg_ms": 3.419, "count": 59, "max_ms": 4.632, "p95_ms": 3.676}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11372}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10686.227, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 158, "h264_decode_error_count": 163, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:44:03+09:00", "last_error_summary": "[h264 @ 0x563f6e1be040] error while decoding MB 0 27, bytestream -7 [in#0/rtsp @ 0x563f6d827e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f6d827e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:44:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 517.403, "last_started_at": "2026-07-01T01:44:06+09:00", "pid": 72004, "process": {"exists": true, "fd_count": 5, "hwm_mb": 325.676, "name": "ffmpeg", "pid": 72004, "rss_mb": 325.676, "state": "S (sleeping)", "threads": 106}, "restart_count": 158, "restart_count_per_hour": 11, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 159}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.241, "sample_count": 2431, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1364751, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:52:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.058, "last_frame_pushed_at": "2026-07-01T01:52:13+09:00", "last_frame_read_age_sec": 30.164, "last_frame_read_at": "2026-07-01T01:52:13+09:00", "mjpeg": {"age_sec": 30.045, "exists": true, "mtime": "2026-07-01T01:52:13.602404+09:00", "mtime_epoch": 1782838333.602404, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:52:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1704.316, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11372}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10672.129, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 158, "h264_decode_error_count": 163, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:44:03+09:00", "last_error_summary": "[h264 @ 0x563f6e1be040] error while decoding MB 0 27, bytestream -7 [in#0/rtsp @ 0x563f6d827e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563f6d827e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:44:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 517.416, "last_started_at": "2026-07-01T01:44:06+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 158, "restart_count_per_hour": 11, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 159}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.241, "sample_count": 2431, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1364751, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:52:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.071, "last_frame_pushed_at": "2026-07-01T01:52:13+09:00", "last_frame_read_age_sec": 30.176, "last_frame_read_at": "2026-07-01T01:52:13+09:00", "mjpeg": {"age_sec": 30.057, "exists": true, "mtime": "2026-07-01T01:52:13.602404+09:00", "mtime_epoch": 1782838333.602404, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:52:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1704.316, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11372}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10387.797, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e434ba7ec0] error while decoding MB 100 30, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 159, "h264_decode_error_count": 164, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:52:43+09:00", "last_error_summary": "[h264 @ 0x55e434ba7ec0] error while decoding MB 100 30, bytestream -7", "last_exit_at": "2026-07-01T01:44:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 517.465, "last_started_at": "2026-07-01T01:44:06+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 158, "restart_count_per_hour": 11, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 159}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.241, "sample_count": 2431, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1364751, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:52:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.12, "last_frame_pushed_at": "2026-07-01T01:52:13+09:00", "last_frame_read_age_sec": 30.226, "last_frame_read_at": "2026-07-01T01:52:13+09:00", "mjpeg": {"age_sec": 30.107, "exists": true, "mtime": "2026-07-01T01:52:13.602404+09:00", "mtime_epoch": 1782838333.602404, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:52:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1704.316, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11372}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10408.059, "limit_mb": 8796093022207.996}, "detail": {"pid": 72370, "restart_count": 159, "start_count": 160}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 159, "h264_decode_error_count": 164, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:52:43+09:00", "last_error_summary": "[h264 @ 0x55e434ba7ec0] error while decoding MB 100 30, bytestream -7", "last_exit_at": "2026-07-01T01:44:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:52:46+09:00", "pid": 72370, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.199, "name": "ffmpeg", "pid": 72370, "rss_mb": 58.199, "state": "S (sleeping)", "threads": 1}, "restart_count": 159, "restart_count_per_hour": 12, "rtsp_timeout_count": 256, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 160}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.276, "sample_count": 2389, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1364751, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:52:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 34.959, "last_frame_pushed_at": "2026-07-01T01:52:13+09:00", "last_frame_read_age_sec": 35.064, "last_frame_read_at": "2026-07-01T01:52:13+09:00", "mjpeg": {"age_sec": 34.945, "exists": true, "mtime": "2026-07-01T01:52:13.602404+09:00", "mtime_epoch": 1782838333.602404, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:52:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1704.316, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11384}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10392.676, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x558236950180] error while decoding MB 28 22, bytestream -5 [in#0/rtsp @ 0x5582368c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582368c8e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 160, "h264_decode_error_count": 165, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:55:45+09:00", "last_error_summary": "[h264 @ 0x558236950180] error while decoding MB 28 22, bytestream -5 [in#0/rtsp @ 0x5582368c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582368c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:44:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 179.451, "last_started_at": "2026-07-01T01:52:46+09:00", "pid": 72370, "process": {"exists": false, "pid": 72370}, "restart_count": 159, "restart_count_per_hour": 11, "rtsp_timeout_count": 258, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 160}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.311, "sample_count": 2098, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1366101, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:55:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-01T01:55:45+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-01T01:55:45+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T01:55:45.938480+09:00", "mtime_epoch": 1782838545.9384797, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95872}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:55:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1705.09, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.689, "pushed": 0.689, "read": 0.747}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.656, "count": 12, "max_ms": 23.083, "p95_ms": 23.083}, "gstreamer_push": {"avg_ms": 1.223, "count": 12, "max_ms": 1.268, "p95_ms": 1.268}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.235, "count": 12, "max_ms": 54.645, "p95_ms": 54.645}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.349, "count": 12, "max_ms": 67.635, "p95_ms": 67.635}, "yolo11_640_fire_11class": {"avg_ms": 61.287, "count": 12, "max_ms": 64.115, "p95_ms": 64.115}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.732, "count": 12, "max_ms": 74.172, "p95_ms": 74.172}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.533, "count": 12, "max_ms": 99.606, "p95_ms": 99.606}, "read_interval": {"avg_ms": 2429.86, "count": 13, "max_ms": 16127.662, "p95_ms": 16127.662}, "triton_wait": {"avg_ms": 69.744, "count": 12, "max_ms": 74.184, "p95_ms": 74.184}, "visualize": {"avg_ms": 3.384, "count": 12, "max_ms": 3.501, "p95_ms": 3.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.413, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11384}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10392.105, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:55:45+09:00", "last_error_summary": "[h264 @ 0x558236950180] error while decoding MB 28 22, bytestream -5 [in#0/rtsp @ 0x5582368c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582368c8e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 160, "h264_decode_error_count": 165, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:55:45+09:00", "last_error_summary": "[h264 @ 0x558236950180] error while decoding MB 28 22, bytestream -5 [in#0/rtsp @ 0x5582368c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582368c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:55:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 179.496, "last_started_at": "2026-07-01T01:52:46+09:00", "pid": 72370, "process": {"exists": false, "pid": 72370}, "restart_count": 159, "restart_count_per_hour": 11, "rtsp_timeout_count": 258, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 160}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.311, "sample_count": 2099, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1366102, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:55:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T01:55:46+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-01T01:55:45+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-01T01:55:45.938480+09:00", "mtime_epoch": 1782838545.9384797, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95872}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:55:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1705.09, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.745, "pushed": 0.745, "read": 0.745}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.579, "count": 13, "max_ms": 23.083, "p95_ms": 23.083}, "gstreamer_push": {"avg_ms": 1.217, "count": 13, "max_ms": 1.268, "p95_ms": 1.268}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.313, "count": 13, "max_ms": 54.645, "p95_ms": 54.645}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.678, "count": 13, "max_ms": 67.635, "p95_ms": 67.635}, "yolo11_640_fire_11class": {"avg_ms": 61.774, "count": 13, "max_ms": 67.616, "p95_ms": 67.616}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.173, "count": 13, "max_ms": 75.468, "p95_ms": 75.468}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.894, "count": 13, "max_ms": 101.226, "p95_ms": 101.226}, "read_interval": {"avg_ms": 2429.86, "count": 13, "max_ms": 16127.662, "p95_ms": 16127.662}, "triton_wait": {"avg_ms": 70.185, "count": 13, "max_ms": 75.48, "p95_ms": 75.48}, "visualize": {"avg_ms": 3.387, "count": 13, "max_ms": 3.501, "p95_ms": 3.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.459, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11384}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10392.148, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 160, "h264_decode_error_count": 165, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:55:45+09:00", "last_error_summary": "[h264 @ 0x558236950180] error while decoding MB 28 22, bytestream -5 [in#0/rtsp @ 0x5582368c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582368c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:55:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 179.502, "last_started_at": "2026-07-01T01:52:46+09:00", "pid": 72370, "process": {"exists": false, "pid": 72370}, "restart_count": 159, "restart_count_per_hour": 11, "rtsp_timeout_count": 258, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 160}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.311, "sample_count": 2098, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1366102, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:55:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T01:55:46+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T01:55:45+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-01T01:55:45.938480+09:00", "mtime_epoch": 1782838545.9384797, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95872}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:55:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1705.09, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.579, "count": 13, "max_ms": 23.083, "p95_ms": 23.083}, "gstreamer_push": {"avg_ms": 1.217, "count": 13, "max_ms": 1.268, "p95_ms": 1.268}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.313, "count": 13, "max_ms": 54.645, "p95_ms": 54.645}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.678, "count": 13, "max_ms": 67.635, "p95_ms": 67.635}, "yolo11_640_fire_11class": {"avg_ms": 61.774, "count": 13, "max_ms": 67.616, "p95_ms": 67.616}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.173, "count": 13, "max_ms": 75.468, "p95_ms": 75.468}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.894, "count": 13, "max_ms": 101.226, "p95_ms": 101.226}, "read_interval": {"avg_ms": 2429.86, "count": 13, "max_ms": 16127.662, "p95_ms": 16127.662}, "triton_wait": {"avg_ms": 70.185, "count": 13, "max_ms": 75.48, "p95_ms": 75.48}, "visualize": {"avg_ms": 3.387, "count": 13, "max_ms": 3.501, "p95_ms": 3.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.464, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11384}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10566.371, "limit_mb": 8796093022207.996}, "detail": {"pid": 72522, "restart_count": 160, "start_count": 161}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 160, "h264_decode_error_count": 165, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:55:45+09:00", "last_error_summary": "[h264 @ 0x558236950180] error while decoding MB 28 22, bytestream -5 [in#0/rtsp @ 0x5582368c8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5582368c8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T01:55:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:55:48+09:00", "pid": 72522, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.051, "name": "ffmpeg", "pid": 72522, "rss_mb": 204.051, "state": "S (sleeping)", "threads": 106}, "restart_count": 160, "restart_count_per_hour": 12, "rtsp_timeout_count": 258, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 161}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.326, "sample_count": 2063, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1366102, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:55:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.176, "last_frame_pushed_at": "2026-07-01T01:55:46+09:00", "last_frame_read_age_sec": 4.279, "last_frame_read_at": "2026-07-01T01:55:45+09:00", "mjpeg": {"age_sec": 4.16, "exists": true, "mtime": "2026-07-01T01:55:46.054482+09:00", "mtime_epoch": 1782838546.054482, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96830}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T01:55:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1705.09, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.601, "pushed": 0.601, "read": 0.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.579, "count": 13, "max_ms": 23.083, "p95_ms": 23.083}, "gstreamer_push": {"avg_ms": 1.217, "count": 13, "max_ms": 1.268, "p95_ms": 1.268}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.313, "count": 13, "max_ms": 54.645, "p95_ms": 54.645}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.678, "count": 13, "max_ms": 67.635, "p95_ms": 67.635}, "yolo11_640_fire_11class": {"avg_ms": 61.774, "count": 13, "max_ms": 67.616, "p95_ms": 67.616}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.173, "count": 13, "max_ms": 75.468, "p95_ms": 75.468}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.894, "count": 13, "max_ms": 101.226, "p95_ms": 101.226}, "read_interval": {"avg_ms": 2429.86, "count": 13, "max_ms": 16127.662, "p95_ms": 16127.662}, "triton_wait": {"avg_ms": 70.185, "count": 13, "max_ms": 75.48, "p95_ms": 75.48}, "visualize": {"avg_ms": 3.387, "count": 13, "max_ms": 3.501, "p95_ms": 3.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.631, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10211}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10448.211, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ff28f45a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f45a00] error while decoding MB 0 26, bytestream 25806 [h264 @ 0x55ff28f866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f866c0] error while decoding MB 0 29, bytestream 1729 [h264 @ 0x55ff29beb900] error while decoding MB 26 43, bytestream -20 [in#0/rtsp @ 0x55ff28f02e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff28f02e00] Error retrieving a packet from ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 361, "h264_decode_error_count": 1178, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:57:30+09:00", "last_error_summary": "[h264 @ 0x55ff28f45a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f45a00] error while decoding MB 0 26, bytestream 25806 [h264 @ 0x55ff28f866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f866c0] error while decoding MB 0 29, bytestream 1729 [h264 @ 0x55ff29beb900] error while decoding MB 26 43, bytestream -20 [in#0/rtsp @ 0x55ff28f02e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff28f02e00] Error retrieving a packet from ", "last_exit_at": "2026-07-01T01:50:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 428.357, "last_started_at": "2026-07-01T01:50:22+09:00", "pid": 72263, "process": {"exists": false, "pid": 72263}, "restart_count": 394, "restart_count_per_hour": 9, "rtsp_timeout_count": 340, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 395}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.482, "sample_count": 2691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1225427, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:57:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.085, "last_frame_pushed_at": "2026-07-01T01:57:30+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-01T01:57:30+09:00", "mjpeg": {"age_sec": 0.069, "exists": true, "mtime": "2026-07-01T01:57:30.464458+09:00", "mtime_epoch": 1782838650.4644575, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 130656}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:57:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4948.852, "name": "celery", "pid": 267, "rss_mb": 4948.852, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.21, "pushed": 4.21, "read": 4.21}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.408, "count": 126, "max_ms": 24.214, "p95_ms": 23.819}, "gstreamer_push": {"avg_ms": 1.235, "count": 126, "max_ms": 2.151, "p95_ms": 1.567}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.674, "count": 126, "max_ms": 43.157, "p95_ms": 41.342}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.029, "count": 126, "max_ms": 70.615, "p95_ms": 68.377}, "yolo11_640_fire_11class": {"avg_ms": 61.71, "count": 126, "max_ms": 68.766, "p95_ms": 67.348}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.995, "count": 126, "max_ms": 78.991, "p95_ms": 77.099}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.427, "count": 126, "max_ms": 108.521, "p95_ms": 105.175}, "read_interval": {"avg_ms": 237.712, "count": 126, "max_ms": 15617.425, "p95_ms": 123.074}, "triton_wait": {"avg_ms": 71.007, "count": 126, "max_ms": 79.004, "p95_ms": 77.111}, "visualize": {"avg_ms": 3.347, "count": 126, "max_ms": 3.569, "p95_ms": 3.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10211}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10448.055, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T01:57:30+09:00", "last_error_summary": "[h264 @ 0x55ff28f45a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f45a00] error while decoding MB 0 26, bytestream 25806 [h264 @ 0x55ff28f866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f866c0] error while decoding MB 0 29, bytestream 1729 [h264 @ 0x55ff29beb900] error while decoding MB 26 43, bytestream -20 [in#0/rtsp @ 0x55ff28f02e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff28f02e00] Error retrieving a packet from ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 361, "h264_decode_error_count": 1178, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:57:30+09:00", "last_error_summary": "[h264 @ 0x55ff28f45a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f45a00] error while decoding MB 0 26, bytestream 25806 [h264 @ 0x55ff28f866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f866c0] error while decoding MB 0 29, bytestream 1729 [h264 @ 0x55ff29beb900] error while decoding MB 26 43, bytestream -20 [in#0/rtsp @ 0x55ff28f02e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff28f02e00] Error retrieving a packet from ", "last_exit_at": "2026-07-01T01:57:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 428.387, "last_started_at": "2026-07-01T01:50:22+09:00", "pid": 72263, "process": {"exists": false, "pid": 72263}, "restart_count": 394, "restart_count_per_hour": 9, "rtsp_timeout_count": 340, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 395}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.482, "sample_count": 2691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1225428, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:57:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T01:57:30+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T01:57:30+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-01T01:57:30.464458+09:00", "mtime_epoch": 1782838650.4644575, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 130656}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:57:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4948.852, "name": "celery", "pid": 267, "rss_mb": 4948.852, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.239, "pushed": 4.239, "read": 4.206}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.504, "count": 127, "max_ms": 32.611, "p95_ms": 23.822}, "gstreamer_push": {"avg_ms": 1.237, "count": 127, "max_ms": 2.151, "p95_ms": 1.567}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.662, "count": 127, "max_ms": 43.157, "p95_ms": 41.342}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.884, "count": 127, "max_ms": 70.615, "p95_ms": 68.377}, "yolo11_640_fire_11class": {"avg_ms": 61.57, "count": 127, "max_ms": 68.766, "p95_ms": 67.348}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.793, "count": 127, "max_ms": 78.991, "p95_ms": 77.099}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.349, "count": 127, "max_ms": 108.521, "p95_ms": 105.175}, "read_interval": {"avg_ms": 237.712, "count": 126, "max_ms": 15617.425, "p95_ms": 123.074}, "triton_wait": {"avg_ms": 70.805, "count": 127, "max_ms": 79.004, "p95_ms": 77.111}, "visualize": {"avg_ms": 3.371, "count": 127, "max_ms": 6.418, "p95_ms": 3.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10211}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10447.914, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 361, "h264_decode_error_count": 1178, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:57:30+09:00", "last_error_summary": "[h264 @ 0x55ff28f45a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f45a00] error while decoding MB 0 26, bytestream 25806 [h264 @ 0x55ff28f866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f866c0] error while decoding MB 0 29, bytestream 1729 [h264 @ 0x55ff29beb900] error while decoding MB 26 43, bytestream -20 [in#0/rtsp @ 0x55ff28f02e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff28f02e00] Error retrieving a packet from ", "last_exit_at": "2026-07-01T01:57:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 428.395, "last_started_at": "2026-07-01T01:50:22+09:00", "pid": 72263, "process": {"exists": false, "pid": 72263}, "restart_count": 394, "restart_count_per_hour": 9, "rtsp_timeout_count": 340, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 395}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.482, "sample_count": 2691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1225428, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:57:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-01T01:57:30+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T01:57:30+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T01:57:30.464458+09:00", "mtime_epoch": 1782838650.4644575, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 130656}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:57:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4948.852, "name": "celery", "pid": 267, "rss_mb": 4948.852, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.238, "pushed": 4.238, "read": 4.205}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.504, "count": 127, "max_ms": 32.611, "p95_ms": 23.822}, "gstreamer_push": {"avg_ms": 1.237, "count": 127, "max_ms": 2.151, "p95_ms": 1.567}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.662, "count": 127, "max_ms": 43.157, "p95_ms": 41.342}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.884, "count": 127, "max_ms": 70.615, "p95_ms": 68.377}, "yolo11_640_fire_11class": {"avg_ms": 61.57, "count": 127, "max_ms": 68.766, "p95_ms": 67.348}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.793, "count": 127, "max_ms": 78.991, "p95_ms": 77.099}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.349, "count": 127, "max_ms": 108.521, "p95_ms": 105.175}, "read_interval": {"avg_ms": 237.712, "count": 126, "max_ms": 15617.425, "p95_ms": 123.074}, "triton_wait": {"avg_ms": 70.805, "count": 127, "max_ms": 79.004, "p95_ms": 77.111}, "visualize": {"avg_ms": 3.371, "count": 127, "max_ms": 6.418, "p95_ms": 3.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10211}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10555.188, "limit_mb": 8796093022207.996}, "detail": {"pid": 72674, "restart_count": 395, "start_count": 396}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 361, "h264_decode_error_count": 1178, "h264_decode_error_count_per_hour": 52, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T01:57:30+09:00", "last_error_summary": "[h264 @ 0x55ff28f45a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f45a00] error while decoding MB 0 26, bytestream 25806 [h264 @ 0x55ff28f866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff28f866c0] error while decoding MB 0 29, bytestream 1729 [h264 @ 0x55ff29beb900] error while decoding MB 26 43, bytestream -20 [in#0/rtsp @ 0x55ff28f02e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff28f02e00] Error retrieving a packet from ", "last_exit_at": "2026-07-01T01:57:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T01:57:32+09:00", "pid": 72674, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.449, "name": "ffmpeg", "pid": 72674, "rss_mb": 138.449, "state": "S (sleeping)", "threads": 106}, "restart_count": 395, "restart_count_per_hour": 10, "rtsp_timeout_count": 340, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 396}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.496, "sample_count": 2645, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1225428, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T01:57:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.1, "last_frame_pushed_at": "2026-07-01T01:57:30+09:00", "last_frame_read_age_sec": 4.192, "last_frame_read_at": "2026-07-01T01:57:30+09:00", "mjpeg": {"age_sec": 4.084, "exists": true, "mtime": "2026-07-01T01:57:30.568460+09:00", "mtime_epoch": 1782838650.5684595, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129164}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T01:57:34+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4948.852, "name": "celery", "pid": 267, "rss_mb": 4948.852, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 91, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.072, "pushed": 3.072, "read": 3.039}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.709, "count": 92, "max_ms": 32.611, "p95_ms": 23.879}, "gstreamer_push": {"avg_ms": 1.253, "count": 92, "max_ms": 2.151, "p95_ms": 1.735}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.172, "count": 92, "max_ms": 43.157, "p95_ms": 41.574}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.299, "count": 92, "max_ms": 70.615, "p95_ms": 68.779}, "yolo11_640_fire_11class": {"avg_ms": 60.756, "count": 92, "max_ms": 68.766, "p95_ms": 66.622}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.631, "count": 92, "max_ms": 78.991, "p95_ms": 76.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.416, "count": 92, "max_ms": 107.556, "p95_ms": 105.175}, "read_interval": {"avg_ms": 284.139, "count": 91, "max_ms": 15617.425, "p95_ms": 123.074}, "triton_wait": {"avg_ms": 69.644, "count": 92, "max_ms": 79.004, "p95_ms": 76.489}, "visualize": {"avg_ms": 3.392, "count": 92, "max_ms": 6.418, "p95_ms": 3.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11437}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10424.078, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55e7f429c200] missing picture in access unit with size 18 [h264 @ 0x55e7f4d03080] no frame! [in#0/rtsp @ 0x55e7f4297e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f4297e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e7f429c840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 161, "h264_decode_error_count": 166, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:07:10+09:00", "last_error_summary": "[NULL @ 0x55e7f429c200] missing picture in access unit with size 18 [h264 @ 0x55e7f4d03080] no frame! [in#0/rtsp @ 0x55e7f4297e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f4297e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e7f429c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T01:55:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 682.468, "last_started_at": "2026-07-01T01:55:48+09:00", "pid": 72522, "process": {"exists": false, "pid": 72522}, "restart_count": 160, "restart_count_per_hour": 9, "rtsp_timeout_count": 260, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 161}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 80.656, "sample_count": 2909, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1372472, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:07:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-01T02:07:10+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-01T02:07:10+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-01T02:07:10.627239+09:00", "mtime_epoch": 1782839230.6272388, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93253}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:07:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1707.926, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.663, "pushed": 0.663, "read": 0.719}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.476, "count": 12, "max_ms": 19.582, "p95_ms": 19.582}, "gstreamer_push": {"avg_ms": 1.253, "count": 12, "max_ms": 1.421, "p95_ms": 1.421}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.999, "count": 12, "max_ms": 54.24, "p95_ms": 54.24}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.423, "count": 12, "max_ms": 67.11, "p95_ms": 67.11}, "yolo11_640_fire_11class": {"avg_ms": 61.542, "count": 12, "max_ms": 66.584, "p95_ms": 66.584}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.069, "count": 12, "max_ms": 75.894, "p95_ms": 75.894}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.67, "count": 12, "max_ms": 102.776, "p95_ms": 102.776}, "read_interval": {"avg_ms": 2459.899, "count": 13, "max_ms": 16792.247, "p95_ms": 16792.247}, "triton_wait": {"avg_ms": 70.08, "count": 12, "max_ms": 75.907, "p95_ms": 75.907}, "visualize": {"avg_ms": 3.423, "count": 12, "max_ms": 3.61, "p95_ms": 3.61}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.089, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11437}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10423.871, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:07:10+09:00", "last_error_summary": "[NULL @ 0x55e7f429c200] missing picture in access unit with size 18 [h264 @ 0x55e7f4d03080] no frame! [in#0/rtsp @ 0x55e7f4297e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f4297e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e7f429c840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 161, "h264_decode_error_count": 166, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:07:10+09:00", "last_error_summary": "[NULL @ 0x55e7f429c200] missing picture in access unit with size 18 [h264 @ 0x55e7f4d03080] no frame! [in#0/rtsp @ 0x55e7f4297e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f4297e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e7f429c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:07:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 682.514, "last_started_at": "2026-07-01T01:55:48+09:00", "pid": 72522, "process": {"exists": false, "pid": 72522}, "restart_count": 160, "restart_count_per_hour": 9, "rtsp_timeout_count": 260, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 161}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 80.668, "sample_count": 2910, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1372473, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:07:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T02:07:10+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-01T02:07:10+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-01T02:07:10.627239+09:00", "mtime_epoch": 1782839230.6272388, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93253}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:07:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1707.926, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.717, "pushed": 0.717, "read": 0.717}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.446, "count": 13, "max_ms": 19.582, "p95_ms": 19.582}, "gstreamer_push": {"avg_ms": 1.256, "count": 13, "max_ms": 1.421, "p95_ms": 1.421}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.712, "count": 13, "max_ms": 54.24, "p95_ms": 54.24}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.522, "count": 13, "max_ms": 67.11, "p95_ms": 67.11}, "yolo11_640_fire_11class": {"avg_ms": 61.737, "count": 13, "max_ms": 66.584, "p95_ms": 66.584}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.263, "count": 13, "max_ms": 75.894, "p95_ms": 75.894}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.829, "count": 13, "max_ms": 102.776, "p95_ms": 102.776}, "read_interval": {"avg_ms": 2459.899, "count": 13, "max_ms": 16792.247, "p95_ms": 16792.247}, "triton_wait": {"avg_ms": 70.275, "count": 13, "max_ms": 75.907, "p95_ms": 75.907}, "visualize": {"avg_ms": 3.421, "count": 13, "max_ms": 3.61, "p95_ms": 3.61}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.135, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11437}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10423.785, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 161, "h264_decode_error_count": 166, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:07:10+09:00", "last_error_summary": "[NULL @ 0x55e7f429c200] missing picture in access unit with size 18 [h264 @ 0x55e7f4d03080] no frame! [in#0/rtsp @ 0x55e7f4297e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f4297e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e7f429c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:07:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 682.522, "last_started_at": "2026-07-01T01:55:48+09:00", "pid": 72522, "process": {"exists": false, "pid": 72522}, "restart_count": 160, "restart_count_per_hour": 9, "rtsp_timeout_count": 260, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 161}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 80.668, "sample_count": 2910, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1372473, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:07:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T02:07:10+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T02:07:10+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-01T02:07:10.627239+09:00", "mtime_epoch": 1782839230.6272388, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93253}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:07:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1707.926, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.717, "pushed": 0.717, "read": 0.717}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.446, "count": 13, "max_ms": 19.582, "p95_ms": 19.582}, "gstreamer_push": {"avg_ms": 1.256, "count": 13, "max_ms": 1.421, "p95_ms": 1.421}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.712, "count": 13, "max_ms": 54.24, "p95_ms": 54.24}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.522, "count": 13, "max_ms": 67.11, "p95_ms": 67.11}, "yolo11_640_fire_11class": {"avg_ms": 61.737, "count": 13, "max_ms": 66.584, "p95_ms": 66.584}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.263, "count": 13, "max_ms": 75.894, "p95_ms": 75.894}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.829, "count": 13, "max_ms": 102.776, "p95_ms": 102.776}, "read_interval": {"avg_ms": 2459.899, "count": 13, "max_ms": 16792.247, "p95_ms": 16792.247}, "triton_wait": {"avg_ms": 70.275, "count": 13, "max_ms": 75.907, "p95_ms": 75.907}, "visualize": {"avg_ms": 3.421, "count": 13, "max_ms": 3.61, "p95_ms": 3.61}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.143, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11437}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10597.859, "limit_mb": 8796093022207.996}, "detail": {"pid": 72781, "restart_count": 161, "start_count": 162}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 161, "h264_decode_error_count": 166, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:07:10+09:00", "last_error_summary": "[NULL @ 0x55e7f429c200] missing picture in access unit with size 18 [h264 @ 0x55e7f4d03080] no frame! [in#0/rtsp @ 0x55e7f4297e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f4297e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55e7f429c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:07:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:07:13+09:00", "pid": 72781, "process": {"exists": true, "fd_count": 5, "hwm_mb": 203.727, "name": "ffmpeg", "pid": 72781, "rss_mb": 203.727, "state": "S (sleeping)", "threads": 106}, "restart_count": 161, "restart_count_per_hour": 10, "rtsp_timeout_count": 260, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 162}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 80.636, "sample_count": 2861, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1372473, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:07:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.592, "last_frame_pushed_at": "2026-07-01T02:07:10+09:00", "last_frame_read_age_sec": 4.693, "last_frame_read_at": "2026-07-01T02:07:10+09:00", "mjpeg": {"age_sec": 4.575, "exists": true, "mtime": "2026-07-01T02:07:10.739241+09:00", "mtime_epoch": 1782839230.739241, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94917}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:07:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1707.926, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.572, "pushed": 0.572, "read": 0.572}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.446, "count": 13, "max_ms": 19.582, "p95_ms": 19.582}, "gstreamer_push": {"avg_ms": 1.256, "count": 13, "max_ms": 1.421, "p95_ms": 1.421}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.712, "count": 13, "max_ms": 54.24, "p95_ms": 54.24}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.522, "count": 13, "max_ms": 67.11, "p95_ms": 67.11}, "yolo11_640_fire_11class": {"avg_ms": 61.737, "count": 13, "max_ms": 66.584, "p95_ms": 66.584}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.263, "count": 13, "max_ms": 75.894, "p95_ms": 75.894}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.829, "count": 13, "max_ms": 102.776, "p95_ms": 102.776}, "read_interval": {"avg_ms": 2459.899, "count": 13, "max_ms": 16792.247, "p95_ms": 16792.247}, "triton_wait": {"avg_ms": 70.275, "count": 13, "max_ms": 75.907, "p95_ms": 75.907}, "visualize": {"avg_ms": 3.421, "count": 13, "max_ms": 3.61, "p95_ms": 3.61}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.722, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10426.855, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55558f0c9400] error while decoding MB 110 32, bytestream -5 [in#0/rtsp @ 0x55558e8c7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55558e8c7e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 162, "h264_decode_error_count": 167, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:10:00+09:00", "last_error_summary": "[h264 @ 0x55558f0c9400] error while decoding MB 110 32, bytestream -5 [in#0/rtsp @ 0x55558e8c7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55558e8c7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:07:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 166.943, "last_started_at": "2026-07-01T02:07:13+09:00", "pid": 72781, "process": {"exists": false, "pid": 72781}, "restart_count": 161, "restart_count_per_hour": 10, "rtsp_timeout_count": 262, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 162}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.145, "sample_count": 2298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1373696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:10:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-01T02:10:00+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T02:10:00+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T02:10:00.210354+09:00", "mtime_epoch": 1782839400.2103536, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94841}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:10:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1708.699, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.686, "pushed": 0.686, "read": 0.743}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.803, "count": 12, "max_ms": 21.814, "p95_ms": 21.814}, "gstreamer_push": {"avg_ms": 1.223, "count": 12, "max_ms": 1.322, "p95_ms": 1.322}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.26, "count": 12, "max_ms": 64.931, "p95_ms": 64.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.383, "count": 12, "max_ms": 71.01, "p95_ms": 71.01}, "yolo11_640_fire_11class": {"avg_ms": 61.049, "count": 12, "max_ms": 69.991, "p95_ms": 69.991}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.852, "count": 12, "max_ms": 79.045, "p95_ms": 79.045}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.852, "count": 12, "max_ms": 105.52, "p95_ms": 105.52}, "read_interval": {"avg_ms": 2774.375, "count": 13, "max_ms": 18622.163, "p95_ms": 18622.163}, "triton_wait": {"avg_ms": 67.997, "count": 12, "max_ms": 79.058, "p95_ms": 79.058}, "visualize": {"avg_ms": 3.383, "count": 12, "max_ms": 3.51, "p95_ms": 3.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.497, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10426.266, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:10:00+09:00", "last_error_summary": "[h264 @ 0x55558f0c9400] error while decoding MB 110 32, bytestream -5 [in#0/rtsp @ 0x55558e8c7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55558e8c7e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 162, "h264_decode_error_count": 167, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:10:00+09:00", "last_error_summary": "[h264 @ 0x55558f0c9400] error while decoding MB 110 32, bytestream -5 [in#0/rtsp @ 0x55558e8c7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55558e8c7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:10:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 166.993, "last_started_at": "2026-07-01T02:07:13+09:00", "pid": 72781, "process": {"exists": false, "pid": 72781}, "restart_count": 161, "restart_count_per_hour": 10, "rtsp_timeout_count": 262, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 162}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.145, "sample_count": 2298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1373697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:10:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T02:10:00+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T02:10:00+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-01T02:10:00.210354+09:00", "mtime_epoch": 1782839400.2103536, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94841}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:10:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1708.699, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.741, "pushed": 0.741, "read": 0.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.701, "count": 13, "max_ms": 21.814, "p95_ms": 21.814}, "gstreamer_push": {"avg_ms": 1.216, "count": 13, "max_ms": 1.322, "p95_ms": 1.322}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.983, "count": 13, "max_ms": 64.931, "p95_ms": 64.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.314, "count": 13, "max_ms": 71.01, "p95_ms": 71.01}, "yolo11_640_fire_11class": {"avg_ms": 61.244, "count": 13, "max_ms": 69.991, "p95_ms": 69.991}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.021, "count": 13, "max_ms": 79.045, "p95_ms": 79.045}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.892, "count": 13, "max_ms": 105.52, "p95_ms": 105.52}, "read_interval": {"avg_ms": 2774.375, "count": 13, "max_ms": 18622.163, "p95_ms": 18622.163}, "triton_wait": {"avg_ms": 68.156, "count": 13, "max_ms": 79.058, "p95_ms": 79.058}, "visualize": {"avg_ms": 3.382, "count": 13, "max_ms": 3.51, "p95_ms": 3.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.547, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10426.266, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 162, "h264_decode_error_count": 167, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:10:00+09:00", "last_error_summary": "[h264 @ 0x55558f0c9400] error while decoding MB 110 32, bytestream -5 [in#0/rtsp @ 0x55558e8c7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55558e8c7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:10:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.0, "last_started_at": "2026-07-01T02:07:13+09:00", "pid": 72781, "process": {"exists": false, "pid": 72781}, "restart_count": 161, "restart_count_per_hour": 10, "rtsp_timeout_count": 262, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 162}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.145, "sample_count": 2298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1373697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:10:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T02:10:00+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-01T02:10:00+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-01T02:10:00.210354+09:00", "mtime_epoch": 1782839400.2103536, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94841}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:10:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1708.699, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.741, "pushed": 0.741, "read": 0.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.701, "count": 13, "max_ms": 21.814, "p95_ms": 21.814}, "gstreamer_push": {"avg_ms": 1.216, "count": 13, "max_ms": 1.322, "p95_ms": 1.322}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.983, "count": 13, "max_ms": 64.931, "p95_ms": 64.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.314, "count": 13, "max_ms": 71.01, "p95_ms": 71.01}, "yolo11_640_fire_11class": {"avg_ms": 61.244, "count": 13, "max_ms": 69.991, "p95_ms": 69.991}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.021, "count": 13, "max_ms": 79.045, "p95_ms": 79.045}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.892, "count": 13, "max_ms": 105.52, "p95_ms": 105.52}, "read_interval": {"avg_ms": 2774.375, "count": 13, "max_ms": 18622.163, "p95_ms": 18622.163}, "triton_wait": {"avg_ms": 68.156, "count": 13, "max_ms": 79.058, "p95_ms": 79.058}, "visualize": {"avg_ms": 3.382, "count": 13, "max_ms": 3.51, "p95_ms": 3.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.553, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10471.445, "limit_mb": 8796093022207.996}, "detail": {"pid": 72933, "restart_count": 162, "start_count": 163}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 162, "h264_decode_error_count": 167, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:10:00+09:00", "last_error_summary": "[h264 @ 0x55558f0c9400] error while decoding MB 110 32, bytestream -5 [in#0/rtsp @ 0x55558e8c7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55558e8c7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:10:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:10:02+09:00", "pid": 72933, "process": {"exists": true, "fd_count": 5, "hwm_mb": 78.16, "name": "ffmpeg", "pid": 72933, "rss_mb": 78.16, "state": "S (sleeping)", "threads": 19}, "restart_count": 162, "restart_count_per_hour": 11, "rtsp_timeout_count": 262, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 163}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.165, "sample_count": 2248, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1373697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:10:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.607, "last_frame_pushed_at": "2026-07-01T02:10:00+09:00", "last_frame_read_age_sec": 4.704, "last_frame_read_at": "2026-07-01T02:10:00+09:00", "mjpeg": {"age_sec": 4.595, "exists": true, "mtime": "2026-07-01T02:10:00.314356+09:00", "mtime_epoch": 1782839400.3143556, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95593}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:10:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1708.699, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.587, "pushed": 0.587, "read": 0.587}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.701, "count": 13, "max_ms": 21.814, "p95_ms": 21.814}, "gstreamer_push": {"avg_ms": 1.216, "count": 13, "max_ms": 1.322, "p95_ms": 1.322}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.983, "count": 13, "max_ms": 64.931, "p95_ms": 64.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.314, "count": 13, "max_ms": 71.01, "p95_ms": 71.01}, "yolo11_640_fire_11class": {"avg_ms": 61.244, "count": 13, "max_ms": 69.991, "p95_ms": 69.991}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.021, "count": 13, "max_ms": 79.045, "p95_ms": 79.045}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.892, "count": 13, "max_ms": 105.52, "p95_ms": 105.52}, "read_interval": {"avg_ms": 2774.375, "count": 13, "max_ms": 18622.163, "p95_ms": 18622.163}, "triton_wait": {"avg_ms": 68.156, "count": 13, "max_ms": 79.058, "p95_ms": 79.058}, "visualize": {"avg_ms": 3.382, "count": 13, "max_ms": 3.51, "p95_ms": 3.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.149, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10431.043, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55ae9356c200] missing picture in access unit with size 18 [h264 @ 0x55ae93fd9a80] no frame! [in#0/rtsp @ 0x55ae93567e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae93567e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae9356c840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 163, "h264_decode_error_count": 168, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:12:50+09:00", "last_error_summary": "[NULL @ 0x55ae9356c200] missing picture in access unit with size 18 [h264 @ 0x55ae93fd9a80] no frame! [in#0/rtsp @ 0x55ae93567e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae93567e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae9356c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:10:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.151, "last_started_at": "2026-07-01T02:10:02+09:00", "pid": 72933, "process": {"exists": false, "pid": 72933}, "restart_count": 162, "restart_count_per_hour": 11, "rtsp_timeout_count": 264, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 163}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.443, "sample_count": 2147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1374920, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:12:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-01T02:12:49+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T02:12:49+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-01T02:12:49.993464+09:00", "mtime_epoch": 1782839569.9934638, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94071}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:12:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1709.215, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.479, "pushed": 0.479, "read": 0.516}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.8, "count": 13, "max_ms": 24.442, "p95_ms": 24.442}, "gstreamer_push": {"avg_ms": 1.224, "count": 13, "max_ms": 1.302, "p95_ms": 1.302}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.075, "count": 13, "max_ms": 47.058, "p95_ms": 47.058}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.631, "count": 13, "max_ms": 51.757, "p95_ms": 51.757}, "yolo11_640_fire_11class": {"avg_ms": 47.131, "count": 13, "max_ms": 51.09, "p95_ms": 51.09}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.34, "count": 13, "max_ms": 52.224, "p95_ms": 52.224}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.506, "count": 13, "max_ms": 81.421, "p95_ms": 81.421}, "read_interval": {"avg_ms": 2177.233, "count": 14, "max_ms": 15943.577, "p95_ms": 15943.577}, "triton_wait": {"avg_ms": 48.609, "count": 13, "max_ms": 52.233, "p95_ms": 52.233}, "visualize": {"avg_ms": 3.458, "count": 13, "max_ms": 3.604, "p95_ms": 3.604}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.135, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10430.742, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:12:50+09:00", "last_error_summary": "[NULL @ 0x55ae9356c200] missing picture in access unit with size 18 [h264 @ 0x55ae93fd9a80] no frame! [in#0/rtsp @ 0x55ae93567e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae93567e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae9356c840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 163, "h264_decode_error_count": 168, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:12:50+09:00", "last_error_summary": "[NULL @ 0x55ae9356c200] missing picture in access unit with size 18 [h264 @ 0x55ae93fd9a80] no frame! [in#0/rtsp @ 0x55ae93567e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae93567e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae9356c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:12:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.178, "last_started_at": "2026-07-01T02:10:02+09:00", "pid": 72933, "process": {"exists": false, "pid": 72933}, "restart_count": 162, "restart_count_per_hour": 11, "rtsp_timeout_count": 264, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 163}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.443, "sample_count": 2148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1374921, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:12:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T02:12:50+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-01T02:12:49+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-01T02:12:49.993464+09:00", "mtime_epoch": 1782839569.9934638, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94071}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:12:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1709.215, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.515, "pushed": 0.515, "read": 0.515}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.022, "count": 14, "max_ms": 24.442, "p95_ms": 24.442}, "gstreamer_push": {"avg_ms": 1.247, "count": 14, "max_ms": 1.539, "p95_ms": 1.539}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.066, "count": 14, "max_ms": 47.058, "p95_ms": 47.058}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.749, "count": 14, "max_ms": 51.757, "p95_ms": 51.757}, "yolo11_640_fire_11class": {"avg_ms": 47.245, "count": 14, "max_ms": 51.09, "p95_ms": 51.09}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.502, "count": 14, "max_ms": 52.224, "p95_ms": 52.224}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.105, "count": 14, "max_ms": 85.889, "p95_ms": 85.889}, "read_interval": {"avg_ms": 2177.233, "count": 14, "max_ms": 15943.577, "p95_ms": 15943.577}, "triton_wait": {"avg_ms": 48.753, "count": 14, "max_ms": 52.233, "p95_ms": 52.233}, "visualize": {"avg_ms": 3.65, "count": 14, "max_ms": 6.147, "p95_ms": 6.147}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.162, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10430.719, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 163, "h264_decode_error_count": 168, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:12:50+09:00", "last_error_summary": "[NULL @ 0x55ae9356c200] missing picture in access unit with size 18 [h264 @ 0x55ae93fd9a80] no frame! [in#0/rtsp @ 0x55ae93567e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae93567e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae9356c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:12:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.185, "last_started_at": "2026-07-01T02:10:02+09:00", "pid": 72933, "process": {"exists": false, "pid": 72933}, "restart_count": 162, "restart_count_per_hour": 11, "rtsp_timeout_count": 264, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 163}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.443, "sample_count": 2148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1374921, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:12:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T02:12:50+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-01T02:12:49+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-01T02:12:49.993464+09:00", "mtime_epoch": 1782839569.9934638, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94071}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:12:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1709.215, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.515, "pushed": 0.515, "read": 0.515}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.022, "count": 14, "max_ms": 24.442, "p95_ms": 24.442}, "gstreamer_push": {"avg_ms": 1.247, "count": 14, "max_ms": 1.539, "p95_ms": 1.539}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.066, "count": 14, "max_ms": 47.058, "p95_ms": 47.058}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.749, "count": 14, "max_ms": 51.757, "p95_ms": 51.757}, "yolo11_640_fire_11class": {"avg_ms": 47.245, "count": 14, "max_ms": 51.09, "p95_ms": 51.09}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.502, "count": 14, "max_ms": 52.224, "p95_ms": 52.224}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.105, "count": 14, "max_ms": 85.889, "p95_ms": 85.889}, "read_interval": {"avg_ms": 2177.233, "count": 14, "max_ms": 15943.577, "p95_ms": 15943.577}, "triton_wait": {"avg_ms": 48.753, "count": 14, "max_ms": 52.233, "p95_ms": 52.233}, "visualize": {"avg_ms": 3.65, "count": 14, "max_ms": 6.147, "p95_ms": 6.147}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.168, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10554.609, "limit_mb": 8796093022207.996}, "detail": {"pid": 73085, "restart_count": 163, "start_count": 164}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 163, "h264_decode_error_count": 168, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:12:50+09:00", "last_error_summary": "[NULL @ 0x55ae9356c200] missing picture in access unit with size 18 [h264 @ 0x55ae93fd9a80] no frame! [in#0/rtsp @ 0x55ae93567e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae93567e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae9356c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:12:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:12:53+09:00", "pid": 73085, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.801, "name": "ffmpeg", "pid": 73085, "rss_mb": 156.801, "state": "R (running)", "threads": 49}, "restart_count": 163, "restart_count_per_hour": 12, "rtsp_timeout_count": 264, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 164}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.462, "sample_count": 2105, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1374921, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:12:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.979, "last_frame_pushed_at": "2026-07-01T02:12:50+09:00", "last_frame_read_age_sec": 5.067, "last_frame_read_at": "2026-07-01T02:12:49+09:00", "mjpeg": {"age_sec": 4.964, "exists": true, "mtime": "2026-07-01T02:12:50.093466+09:00", "mtime_epoch": 1782839570.0934656, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94631}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:12:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1709.215, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.591, "pushed": 0.591, "read": 0.591}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.122, "count": 13, "max_ms": 24.442, "p95_ms": 24.442}, "gstreamer_push": {"avg_ms": 1.253, "count": 13, "max_ms": 1.539, "p95_ms": 1.539}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.085, "count": 13, "max_ms": 47.058, "p95_ms": 47.058}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.95, "count": 13, "max_ms": 51.757, "p95_ms": 51.757}, "yolo11_640_fire_11class": {"avg_ms": 47.366, "count": 13, "max_ms": 51.09, "p95_ms": 51.09}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.634, "count": 13, "max_ms": 52.224, "p95_ms": 52.224}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.431, "count": 13, "max_ms": 85.889, "p95_ms": 85.889}, "read_interval": {"avg_ms": 2081.903, "count": 13, "max_ms": 15943.577, "p95_ms": 15943.577}, "triton_wait": {"avg_ms": 48.903, "count": 13, "max_ms": 52.233, "p95_ms": 52.233}, "visualize": {"avg_ms": 3.7, "count": 13, "max_ms": 6.147, "p95_ms": 6.147}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.012, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11469}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10736.641, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 163, "h264_decode_error_count": 168, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:12:50+09:00", "last_error_summary": "[NULL @ 0x55ae9356c200] missing picture in access unit with size 18 [h264 @ 0x55ae93fd9a80] no frame! [in#0/rtsp @ 0x55ae93567e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae93567e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae9356c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:12:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.891, "last_started_at": "2026-07-01T02:12:53+09:00", "pid": 73085, "process": {"exists": true, "fd_count": 5, "hwm_mb": 326.113, "name": "ffmpeg", "pid": 73085, "rss_mb": 326.113, "state": "S (sleeping)", "threads": 106}, "restart_count": 163, "restart_count_per_hour": 12, "rtsp_timeout_count": 264, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 164}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.833, "sample_count": 2063, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1376355, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:15:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.058, "last_frame_pushed_at": "2026-07-01T02:15:37+09:00", "last_frame_read_age_sec": 30.159, "last_frame_read_at": "2026-07-01T02:15:37+09:00", "mjpeg": {"age_sec": 30.042, "exists": true, "mtime": "2026-07-01T02:15:37.904533+09:00", "mtime_epoch": 1782839737.9045331, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99073}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:16:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1709.988, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11469}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10722.676, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 163, "h264_decode_error_count": 168, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:12:50+09:00", "last_error_summary": "[NULL @ 0x55ae9356c200] missing picture in access unit with size 18 [h264 @ 0x55ae93fd9a80] no frame! [in#0/rtsp @ 0x55ae93567e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae93567e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae9356c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:12:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.903, "last_started_at": "2026-07-01T02:12:53+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 163, "restart_count_per_hour": 12, "rtsp_timeout_count": 264, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 164}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.833, "sample_count": 2063, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1376355, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:15:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.071, "last_frame_pushed_at": "2026-07-01T02:15:37+09:00", "last_frame_read_age_sec": 30.171, "last_frame_read_at": "2026-07-01T02:15:37+09:00", "mjpeg": {"age_sec": 30.055, "exists": true, "mtime": "2026-07-01T02:15:37.904533+09:00", "mtime_epoch": 1782839737.9045331, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99073}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:16:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1709.988, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11469}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10438.645, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560486914c00] error while decoding MB 88 28, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 164, "h264_decode_error_count": 169, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:16:08+09:00", "last_error_summary": "[h264 @ 0x560486914c00] error while decoding MB 88 28, bytestream -9", "last_exit_at": "2026-07-01T02:12:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.953, "last_started_at": "2026-07-01T02:12:53+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 163, "restart_count_per_hour": 12, "rtsp_timeout_count": 264, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 164}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.833, "sample_count": 2063, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1376355, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:15:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.12, "last_frame_pushed_at": "2026-07-01T02:15:37+09:00", "last_frame_read_age_sec": 30.221, "last_frame_read_at": "2026-07-01T02:15:37+09:00", "mjpeg": {"age_sec": 30.104, "exists": true, "mtime": "2026-07-01T02:15:37.904533+09:00", "mtime_epoch": 1782839737.9045331, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99073}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:16:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1709.988, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11469}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10459.223, "limit_mb": 8796093022207.996}, "detail": {"pid": 73237, "restart_count": 164, "start_count": 165}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 164, "h264_decode_error_count": 169, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:16:08+09:00", "last_error_summary": "[h264 @ 0x560486914c00] error while decoding MB 88 28, bytestream -9", "last_exit_at": "2026-07-01T02:12:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:16:10+09:00", "pid": 73237, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.371, "name": "ffmpeg", "pid": 73237, "rss_mb": 57.371, "state": "S (sleeping)", "threads": 1}, "restart_count": 164, "restart_count_per_hour": 13, "rtsp_timeout_count": 264, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 165}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.842, "sample_count": 2021, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1376355, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:15:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 34.94, "last_frame_pushed_at": "2026-07-01T02:15:37+09:00", "last_frame_read_age_sec": 35.041, "last_frame_read_at": "2026-07-01T02:15:37+09:00", "mjpeg": {"age_sec": 34.925, "exists": true, "mtime": "2026-07-01T02:15:37.904533+09:00", "mtime_epoch": 1782839737.9045331, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99073}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:16:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1709.988, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10307}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10459.711, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559dad794100] error while decoding MB 5 25, bytestream -8 [h264 @ 0x559dace617c0] cabac decode of qscale diff failed at 8 25 [h264 @ 0x559dace617c0] error while decoding MB 8 25, bytestream 3247 [h264 @ 0x559dacf295c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf295c0] error while decoding MB 0 18, bytestream 684 [h264 @ 0x559dacf15400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf15400] error while decoding MB 0 3, bytestream 35627 ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 362, "h264_decode_error_count": 1183, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:17:28+09:00", "last_error_summary": "[h264 @ 0x559dad794100] error while decoding MB 5 25, bytestream -8 [h264 @ 0x559dace617c0] cabac decode of qscale diff failed at 8 25 [h264 @ 0x559dace617c0] error while decoding MB 8 25, bytestream 3247 [h264 @ 0x559dacf295c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf295c0] error while decoding MB 0 18, bytestream 684 [h264 @ 0x559dacf15400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf15400] error while decoding MB 0 3, bytestream 35627 ", "last_exit_at": "2026-07-01T01:57:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 1195.87, "last_started_at": "2026-07-01T01:57:32+09:00", "pid": 72674, "process": {"exists": false, "pid": 72674}, "restart_count": 395, "restart_count_per_hour": 7, "rtsp_timeout_count": 342, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 396}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.621, "sample_count": 2701, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1236914, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:17:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-01T02:17:28+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T02:17:28+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T02:17:28.462552+09:00", "mtime_epoch": 1782839848.4625518, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123961}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:17:28+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4954.523, "name": "celery", "pid": 267, "rss_mb": 4954.523, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 108, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.612, "pushed": 3.612, "read": 3.612}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.684, "count": 108, "max_ms": 30.732, "p95_ms": 23.49}, "gstreamer_push": {"avg_ms": 1.273, "count": 108, "max_ms": 1.503, "p95_ms": 1.382}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.388, "count": 108, "max_ms": 43.216, "p95_ms": 41.34}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.203, "count": 108, "max_ms": 67.559, "p95_ms": 65.64}, "yolo11_640_fire_11class": {"avg_ms": 51.269, "count": 108, "max_ms": 67.024, "p95_ms": 64.906}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.862, "count": 108, "max_ms": 77.205, "p95_ms": 74.609}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.853, "count": 108, "max_ms": 110.186, "p95_ms": 104.012}, "read_interval": {"avg_ms": 277.164, "count": 108, "max_ms": 19107.26, "p95_ms": 120.049}, "triton_wait": {"avg_ms": 55.939, "count": 108, "max_ms": 77.218, "p95_ms": 74.622}, "visualize": {"avg_ms": 3.405, "count": 108, "max_ms": 4.398, "p95_ms": 3.615}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10307}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10459.012, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:17:28+09:00", "last_error_summary": "[h264 @ 0x559dad794100] error while decoding MB 5 25, bytestream -8 [h264 @ 0x559dace617c0] cabac decode of qscale diff failed at 8 25 [h264 @ 0x559dace617c0] error while decoding MB 8 25, bytestream 3247 [h264 @ 0x559dacf295c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf295c0] error while decoding MB 0 18, bytestream 684 [h264 @ 0x559dacf15400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf15400] error while decoding MB 0 3, bytestream 35627 ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 362, "h264_decode_error_count": 1183, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:17:28+09:00", "last_error_summary": "[h264 @ 0x559dad794100] error while decoding MB 5 25, bytestream -8 [h264 @ 0x559dace617c0] cabac decode of qscale diff failed at 8 25 [h264 @ 0x559dace617c0] error while decoding MB 8 25, bytestream 3247 [h264 @ 0x559dacf295c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf295c0] error while decoding MB 0 18, bytestream 684 [h264 @ 0x559dacf15400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf15400] error while decoding MB 0 3, bytestream 35627 ", "last_exit_at": "2026-07-01T02:17:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 1195.896, "last_started_at": "2026-07-01T01:57:32+09:00", "pid": 72674, "process": {"exists": false, "pid": 72674}, "restart_count": 395, "restart_count_per_hour": 7, "rtsp_timeout_count": 342, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 396}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.621, "sample_count": 2702, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1236915, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:17:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T02:17:28+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-01T02:17:28+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-01T02:17:28.462552+09:00", "mtime_epoch": 1782839848.4625518, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123961}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:17:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4954.523, "name": "celery", "pid": 267, "rss_mb": 4954.523, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.642, "pushed": 3.642, "read": 3.609}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.723, "count": 109, "max_ms": 30.732, "p95_ms": 23.501}, "gstreamer_push": {"avg_ms": 1.276, "count": 109, "max_ms": 1.533, "p95_ms": 1.388}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.372, "count": 109, "max_ms": 43.216, "p95_ms": 41.34}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.137, "count": 109, "max_ms": 67.559, "p95_ms": 65.64}, "yolo11_640_fire_11class": {"avg_ms": 51.203, "count": 109, "max_ms": 67.024, "p95_ms": 64.906}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.768, "count": 109, "max_ms": 77.205, "p95_ms": 74.609}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.828, "count": 109, "max_ms": 110.186, "p95_ms": 104.012}, "read_interval": {"avg_ms": 277.164, "count": 108, "max_ms": 19107.26, "p95_ms": 120.049}, "triton_wait": {"avg_ms": 55.844, "count": 109, "max_ms": 77.218, "p95_ms": 74.622}, "visualize": {"avg_ms": 3.431, "count": 109, "max_ms": 6.243, "p95_ms": 3.633}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10307}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10458.875, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 362, "h264_decode_error_count": 1183, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:17:28+09:00", "last_error_summary": "[h264 @ 0x559dad794100] error while decoding MB 5 25, bytestream -8 [h264 @ 0x559dace617c0] cabac decode of qscale diff failed at 8 25 [h264 @ 0x559dace617c0] error while decoding MB 8 25, bytestream 3247 [h264 @ 0x559dacf295c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf295c0] error while decoding MB 0 18, bytestream 684 [h264 @ 0x559dacf15400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf15400] error while decoding MB 0 3, bytestream 35627 ", "last_exit_at": "2026-07-01T02:17:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 1195.904, "last_started_at": "2026-07-01T01:57:32+09:00", "pid": 72674, "process": {"exists": false, "pid": 72674}, "restart_count": 395, "restart_count_per_hour": 7, "rtsp_timeout_count": 342, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 396}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.621, "sample_count": 2702, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1236915, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:17:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T02:17:28+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T02:17:28+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-01T02:17:28.462552+09:00", "mtime_epoch": 1782839848.4625518, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123961}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:17:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4954.523, "name": "celery", "pid": 267, "rss_mb": 4954.523, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.641, "pushed": 3.641, "read": 3.608}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.723, "count": 109, "max_ms": 30.732, "p95_ms": 23.501}, "gstreamer_push": {"avg_ms": 1.276, "count": 109, "max_ms": 1.533, "p95_ms": 1.388}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.372, "count": 109, "max_ms": 43.216, "p95_ms": 41.34}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.137, "count": 109, "max_ms": 67.559, "p95_ms": 65.64}, "yolo11_640_fire_11class": {"avg_ms": 51.203, "count": 109, "max_ms": 67.024, "p95_ms": 64.906}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.768, "count": 109, "max_ms": 77.205, "p95_ms": 74.609}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.828, "count": 109, "max_ms": 110.186, "p95_ms": 104.012}, "read_interval": {"avg_ms": 277.164, "count": 108, "max_ms": 19107.26, "p95_ms": 120.049}, "triton_wait": {"avg_ms": 55.844, "count": 109, "max_ms": 77.218, "p95_ms": 74.622}, "visualize": {"avg_ms": 3.431, "count": 109, "max_ms": 6.243, "p95_ms": 3.633}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10307}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10567.875, "limit_mb": 8796093022207.996}, "detail": {"pid": 73389, "restart_count": 396, "start_count": 397}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 362, "h264_decode_error_count": 1183, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:17:28+09:00", "last_error_summary": "[h264 @ 0x559dad794100] error while decoding MB 5 25, bytestream -8 [h264 @ 0x559dace617c0] cabac decode of qscale diff failed at 8 25 [h264 @ 0x559dace617c0] error while decoding MB 8 25, bytestream 3247 [h264 @ 0x559dacf295c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf295c0] error while decoding MB 0 18, bytestream 684 [h264 @ 0x559dacf15400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559dacf15400] error while decoding MB 0 3, bytestream 35627 ", "last_exit_at": "2026-07-01T02:17:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:17:30+09:00", "pid": 73389, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.215, "name": "ffmpeg", "pid": 73389, "rss_mb": 139.215, "state": "S (sleeping)", "threads": 106}, "restart_count": 396, "restart_count_per_hour": 8, "rtsp_timeout_count": 342, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 397}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.644, "sample_count": 2656, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1236915, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:17:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.072, "last_frame_pushed_at": "2026-07-01T02:17:28+09:00", "last_frame_read_age_sec": 4.158, "last_frame_read_at": "2026-07-01T02:17:28+09:00", "mjpeg": {"age_sec": 4.055, "exists": true, "mtime": "2026-07-01T02:17:28.558553+09:00", "mtime_epoch": 1782839848.5585535, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123888}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:17:32+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4954.523, "name": "celery", "pid": 267, "rss_mb": 4954.523, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 67, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.234, "pushed": 2.234, "read": 2.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.388, "count": 67, "max_ms": 26.018, "p95_ms": 23.469}, "gstreamer_push": {"avg_ms": 1.263, "count": 67, "max_ms": 1.533, "p95_ms": 1.349}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.958, "count": 67, "max_ms": 43.216, "p95_ms": 41.629}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.222, "count": 67, "max_ms": 67.559, "p95_ms": 66.344}, "yolo11_640_fire_11class": {"avg_ms": 53.3, "count": 67, "max_ms": 67.024, "p95_ms": 65.87}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.755, "count": 67, "max_ms": 77.205, "p95_ms": 75.976}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.472, "count": 67, "max_ms": 106.293, "p95_ms": 104.702}, "read_interval": {"avg_ms": 386.99, "count": 67, "max_ms": 19107.26, "p95_ms": 120.049}, "triton_wait": {"avg_ms": 58.871, "count": 67, "max_ms": 77.218, "p95_ms": 75.989}, "visualize": {"avg_ms": 3.41, "count": 67, "max_ms": 6.243, "p95_ms": 3.584}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10411.016, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55da17e1b800] error while decoding MB 42 15, bytestream -5 [in#0/rtsp @ 0x55da1788fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55da1788fe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 165, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:19:36+09:00", "last_error_summary": "[h264 @ 0x55da17e1b800] error while decoding MB 42 15, bytestream -5 [in#0/rtsp @ 0x55da1788fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55da1788fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:12:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 205.838, "last_started_at": "2026-07-01T02:16:10+09:00", "pid": 73237, "process": {"exists": false, "pid": 73237}, "restart_count": 164, "restart_count_per_hour": 12, "rtsp_timeout_count": 266, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 165}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.692, "sample_count": 2117, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1377946, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:19:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T02:19:36+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-01T02:19:36+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T02:19:36.604890+09:00", "mtime_epoch": 1782839976.6048896, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92394}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:19:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1710.762, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.692, "pushed": 0.692, "read": 0.75}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.919, "count": 12, "max_ms": 25.608, "p95_ms": 25.608}, "gstreamer_push": {"avg_ms": 1.235, "count": 12, "max_ms": 1.334, "p95_ms": 1.334}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.89, "count": 12, "max_ms": 46.196, "p95_ms": 46.196}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.076, "count": 12, "max_ms": 56.334, "p95_ms": 56.334}, "yolo11_640_fire_11class": {"avg_ms": 48.845, "count": 12, "max_ms": 56.345, "p95_ms": 56.345}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.899, "count": 12, "max_ms": 57.478, "p95_ms": 57.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.968, "count": 12, "max_ms": 90.5, "p95_ms": 90.5}, "read_interval": {"avg_ms": 2574.349, "count": 13, "max_ms": 16270.998, "p95_ms": 16270.998}, "triton_wait": {"avg_ms": 49.994, "count": 12, "max_ms": 57.489, "p95_ms": 57.489}, "visualize": {"avg_ms": 3.417, "count": 12, "max_ms": 3.712, "p95_ms": 3.712}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.343, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10410.711, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:19:36+09:00", "last_error_summary": "[h264 @ 0x55da17e1b800] error while decoding MB 42 15, bytestream -5 [in#0/rtsp @ 0x55da1788fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55da1788fe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 165, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:19:36+09:00", "last_error_summary": "[h264 @ 0x55da17e1b800] error while decoding MB 42 15, bytestream -5 [in#0/rtsp @ 0x55da1788fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55da1788fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:19:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 205.864, "last_started_at": "2026-07-01T02:16:10+09:00", "pid": 73237, "process": {"exists": false, "pid": 73237}, "restart_count": 164, "restart_count_per_hour": 12, "rtsp_timeout_count": 266, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 165}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.692, "sample_count": 2118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1377947, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:19:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T02:19:36+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-01T02:19:36+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T02:19:36.604890+09:00", "mtime_epoch": 1782839976.6048896, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92394}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:19:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1710.762, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.748, "pushed": 0.748, "read": 0.748}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.055, "count": 13, "max_ms": 25.608, "p95_ms": 25.608}, "gstreamer_push": {"avg_ms": 1.248, "count": 13, "max_ms": 1.413, "p95_ms": 1.413}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.664, "count": 13, "max_ms": 46.196, "p95_ms": 46.196}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.872, "count": 13, "max_ms": 56.334, "p95_ms": 56.334}, "yolo11_640_fire_11class": {"avg_ms": 48.658, "count": 13, "max_ms": 56.345, "p95_ms": 56.345}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.717, "count": 13, "max_ms": 57.478, "p95_ms": 57.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.175, "count": 13, "max_ms": 90.5, "p95_ms": 90.5}, "read_interval": {"avg_ms": 2574.349, "count": 13, "max_ms": 16270.998, "p95_ms": 16270.998}, "triton_wait": {"avg_ms": 49.805, "count": 13, "max_ms": 57.489, "p95_ms": 57.489}, "visualize": {"avg_ms": 3.644, "count": 13, "max_ms": 6.371, "p95_ms": 6.371}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.369, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10410.703, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 165, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:19:36+09:00", "last_error_summary": "[h264 @ 0x55da17e1b800] error while decoding MB 42 15, bytestream -5 [in#0/rtsp @ 0x55da1788fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55da1788fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:19:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 205.871, "last_started_at": "2026-07-01T02:16:10+09:00", "pid": 73237, "process": {"exists": false, "pid": 73237}, "restart_count": 164, "restart_count_per_hour": 12, "rtsp_timeout_count": 266, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 165}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.692, "sample_count": 2118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1377947, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:19:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T02:19:36+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T02:19:36+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-01T02:19:36.604890+09:00", "mtime_epoch": 1782839976.6048896, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92394}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:19:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1710.762, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.748, "pushed": 0.748, "read": 0.748}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.055, "count": 13, "max_ms": 25.608, "p95_ms": 25.608}, "gstreamer_push": {"avg_ms": 1.248, "count": 13, "max_ms": 1.413, "p95_ms": 1.413}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.664, "count": 13, "max_ms": 46.196, "p95_ms": 46.196}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.872, "count": 13, "max_ms": 56.334, "p95_ms": 56.334}, "yolo11_640_fire_11class": {"avg_ms": 48.658, "count": 13, "max_ms": 56.345, "p95_ms": 56.345}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.717, "count": 13, "max_ms": 57.478, "p95_ms": 57.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.175, "count": 13, "max_ms": 90.5, "p95_ms": 90.5}, "read_interval": {"avg_ms": 2574.349, "count": 13, "max_ms": 16270.998, "p95_ms": 16270.998}, "triton_wait": {"avg_ms": 49.805, "count": 13, "max_ms": 57.489, "p95_ms": 57.489}, "visualize": {"avg_ms": 3.644, "count": 13, "max_ms": 6.371, "p95_ms": 6.371}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.376, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10584.957, "limit_mb": 8796093022207.996}, "detail": {"pid": 73496, "restart_count": 165, "start_count": 166}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 165, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:19:36+09:00", "last_error_summary": "[h264 @ 0x55da17e1b800] error while decoding MB 42 15, bytestream -5 [in#0/rtsp @ 0x55da1788fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55da1788fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:19:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:19:39+09:00", "pid": 73496, "process": {"exists": true, "fd_count": 5, "hwm_mb": 203.93, "name": "ffmpeg", "pid": 73496, "rss_mb": 203.93, "state": "S (sleeping)", "threads": 106}, "restart_count": 165, "restart_count_per_hour": 13, "rtsp_timeout_count": 266, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 166}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.785, "sample_count": 2078, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1377947, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:19:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.651, "last_frame_pushed_at": "2026-07-01T02:19:36+09:00", "last_frame_read_age_sec": 4.736, "last_frame_read_at": "2026-07-01T02:19:36+09:00", "mjpeg": {"age_sec": 4.638, "exists": true, "mtime": "2026-07-01T02:19:36.696891+09:00", "mtime_epoch": 1782839976.6968913, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93233}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:19:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1710.762, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.591, "pushed": 0.591, "read": 0.591}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.055, "count": 13, "max_ms": 25.608, "p95_ms": 25.608}, "gstreamer_push": {"avg_ms": 1.248, "count": 13, "max_ms": 1.413, "p95_ms": 1.413}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.664, "count": 13, "max_ms": 46.196, "p95_ms": 46.196}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.872, "count": 13, "max_ms": 56.334, "p95_ms": 56.334}, "yolo11_640_fire_11class": {"avg_ms": 48.658, "count": 13, "max_ms": 56.345, "p95_ms": 56.345}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.717, "count": 13, "max_ms": 57.478, "p95_ms": 57.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.175, "count": 13, "max_ms": 90.5, "p95_ms": 90.5}, "read_interval": {"avg_ms": 2574.349, "count": 13, "max_ms": 16270.998, "p95_ms": 16270.998}, "triton_wait": {"avg_ms": 49.805, "count": 13, "max_ms": 57.489, "p95_ms": 57.489}, "visualize": {"avg_ms": 3.644, "count": 13, "max_ms": 6.371, "p95_ms": 6.371}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.012, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10465.055, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556c7fa3f780] error while decoding MB 64 24, bytestream -7 [h264 @ 0x556c7edc1bc0] left block unavailable for requested intra mode [h264 @ 0x556c7edc1bc0] error while decoding MB 0 8, bytestream 4681 [h264 @ 0x556c7eeb0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c7eeb0880] error while decoding MB 0 27, bytestream 1200 [in#0/rtsp @ 0x556c7ed51e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c7ed51e00] Error retrieving a packet from demuxer: ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 363, "h264_decode_error_count": 1186, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:31:09+09:00", "last_error_summary": "[h264 @ 0x556c7fa3f780] error while decoding MB 64 24, bytestream -7 [h264 @ 0x556c7edc1bc0] left block unavailable for requested intra mode [h264 @ 0x556c7edc1bc0] error while decoding MB 0 8, bytestream 4681 [h264 @ 0x556c7eeb0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c7eeb0880] error while decoding MB 0 27, bytestream 1200 [in#0/rtsp @ 0x556c7ed51e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c7ed51e00] Error retrieving a packet from demuxer: ", "last_exit_at": "2026-07-01T02:17:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 819.104, "last_started_at": "2026-07-01T02:17:30+09:00", "pid": 73389, "process": {"exists": false, "pid": 73389}, "restart_count": 396, "restart_count_per_hour": 6, "rtsp_timeout_count": 344, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 397}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.859, "sample_count": 2814, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1244615, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:31:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T02:31:09+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-01T02:31:09+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-01T02:31:09.652880+09:00", "mtime_epoch": 1782840669.65288, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 131333}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:31:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4958.133, "name": "celery", "pid": 267, "rss_mb": 4958.133, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 163, "pushed_frames": 163, "read_frames": 164, "select_timeouts": 13, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.435, "pushed": 5.435, "read": 5.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.457, "count": 163, "max_ms": 26.744, "p95_ms": 23.432}, "gstreamer_push": {"avg_ms": 1.227, "count": 163, "max_ms": 1.433, "p95_ms": 1.323}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.204, "count": 163, "max_ms": 43.841, "p95_ms": 35.016}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.797, "count": 163, "max_ms": 69.776, "p95_ms": 67.915}, "yolo11_640_fire_11class": {"avg_ms": 52.399, "count": 163, "max_ms": 71.52, "p95_ms": 68.437}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.622, "count": 163, "max_ms": 78.873, "p95_ms": 76.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.111, "count": 163, "max_ms": 107.225, "p95_ms": 105.491}, "read_interval": {"avg_ms": 183.026, "count": 164, "max_ms": 10166.052, "p95_ms": 121.329}, "triton_wait": {"avg_ms": 56.633, "count": 163, "max_ms": 78.885, "p95_ms": 76.533}, "visualize": {"avg_ms": 3.4, "count": 163, "max_ms": 4.032, "p95_ms": 3.605}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10464.719, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:31:09+09:00", "last_error_summary": "[h264 @ 0x556c7fa3f780] error while decoding MB 64 24, bytestream -7 [h264 @ 0x556c7edc1bc0] left block unavailable for requested intra mode [h264 @ 0x556c7edc1bc0] error while decoding MB 0 8, bytestream 4681 [h264 @ 0x556c7eeb0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c7eeb0880] error while decoding MB 0 27, bytestream 1200 [in#0/rtsp @ 0x556c7ed51e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c7ed51e00] Error retrieving a packet from demuxer: ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 363, "h264_decode_error_count": 1186, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:31:09+09:00", "last_error_summary": "[h264 @ 0x556c7fa3f780] error while decoding MB 64 24, bytestream -7 [h264 @ 0x556c7edc1bc0] left block unavailable for requested intra mode [h264 @ 0x556c7edc1bc0] error while decoding MB 0 8, bytestream 4681 [h264 @ 0x556c7eeb0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c7eeb0880] error while decoding MB 0 27, bytestream 1200 [in#0/rtsp @ 0x556c7ed51e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c7ed51e00] Error retrieving a packet from demuxer: ", "last_exit_at": "2026-07-01T02:31:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 819.132, "last_started_at": "2026-07-01T02:17:30+09:00", "pid": 73389, "process": {"exists": false, "pid": 73389}, "restart_count": 396, "restart_count_per_hour": 6, "rtsp_timeout_count": 344, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 397}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.859, "sample_count": 2814, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1244616, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:31:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T02:31:09+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-01T02:31:09+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T02:31:09.652880+09:00", "mtime_epoch": 1782840669.65288, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 131333}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:31:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4958.133, "name": "celery", "pid": 267, "rss_mb": 4958.133, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 164, "pushed_frames": 164, "read_frames": 163, "select_timeouts": 13, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.477, "pushed": 5.477, "read": 5.444}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.489, "count": 164, "max_ms": 26.744, "p95_ms": 23.496}, "gstreamer_push": {"avg_ms": 1.228, "count": 164, "max_ms": 1.433, "p95_ms": 1.328}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.201, "count": 164, "max_ms": 43.841, "p95_ms": 35.016}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.765, "count": 164, "max_ms": 69.776, "p95_ms": 67.915}, "yolo11_640_fire_11class": {"avg_ms": 52.361, "count": 164, "max_ms": 71.52, "p95_ms": 68.437}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.567, "count": 164, "max_ms": 78.873, "p95_ms": 76.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.104, "count": 164, "max_ms": 107.225, "p95_ms": 105.491}, "read_interval": {"avg_ms": 183.565, "count": 163, "max_ms": 10166.052, "p95_ms": 121.329}, "triton_wait": {"avg_ms": 56.579, "count": 164, "max_ms": 78.885, "p95_ms": 76.533}, "visualize": {"avg_ms": 3.414, "count": 164, "max_ms": 5.68, "p95_ms": 3.607}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10464.504, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 363, "h264_decode_error_count": 1186, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:31:09+09:00", "last_error_summary": "[h264 @ 0x556c7fa3f780] error while decoding MB 64 24, bytestream -7 [h264 @ 0x556c7edc1bc0] left block unavailable for requested intra mode [h264 @ 0x556c7edc1bc0] error while decoding MB 0 8, bytestream 4681 [h264 @ 0x556c7eeb0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c7eeb0880] error while decoding MB 0 27, bytestream 1200 [in#0/rtsp @ 0x556c7ed51e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c7ed51e00] Error retrieving a packet from demuxer: ", "last_exit_at": "2026-07-01T02:31:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 819.142, "last_started_at": "2026-07-01T02:17:30+09:00", "pid": 73389, "process": {"exists": false, "pid": 73389}, "restart_count": 396, "restart_count_per_hour": 6, "rtsp_timeout_count": 344, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 397}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.859, "sample_count": 2814, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1244616, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:31:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-01T02:31:09+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-01T02:31:09+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-01T02:31:09.652880+09:00", "mtime_epoch": 1782840669.65288, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 131333}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:31:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4958.133, "name": "celery", "pid": 267, "rss_mb": 4958.133, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 164, "pushed_frames": 164, "read_frames": 163, "select_timeouts": 13, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.475, "pushed": 5.475, "read": 5.442}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.489, "count": 164, "max_ms": 26.744, "p95_ms": 23.496}, "gstreamer_push": {"avg_ms": 1.228, "count": 164, "max_ms": 1.433, "p95_ms": 1.328}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.201, "count": 164, "max_ms": 43.841, "p95_ms": 35.016}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.765, "count": 164, "max_ms": 69.776, "p95_ms": 67.915}, "yolo11_640_fire_11class": {"avg_ms": 52.361, "count": 164, "max_ms": 71.52, "p95_ms": 68.437}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.567, "count": 164, "max_ms": 78.873, "p95_ms": 76.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.104, "count": 164, "max_ms": 107.225, "p95_ms": 105.491}, "read_interval": {"avg_ms": 183.565, "count": 163, "max_ms": 10166.052, "p95_ms": 121.329}, "triton_wait": {"avg_ms": 56.579, "count": 164, "max_ms": 78.885, "p95_ms": 76.533}, "visualize": {"avg_ms": 3.414, "count": 164, "max_ms": 5.68, "p95_ms": 3.607}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10575.109, "limit_mb": 8796093022207.996}, "detail": {"pid": 73648, "restart_count": 397, "start_count": 398}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 363, "h264_decode_error_count": 1186, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:31:09+09:00", "last_error_summary": "[h264 @ 0x556c7fa3f780] error while decoding MB 64 24, bytestream -7 [h264 @ 0x556c7edc1bc0] left block unavailable for requested intra mode [h264 @ 0x556c7edc1bc0] error while decoding MB 0 8, bytestream 4681 [h264 @ 0x556c7eeb0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c7eeb0880] error while decoding MB 0 27, bytestream 1200 [in#0/rtsp @ 0x556c7ed51e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c7ed51e00] Error retrieving a packet from demuxer: ", "last_exit_at": "2026-07-01T02:31:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:31:12+09:00", "pid": 73648, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.613, "name": "ffmpeg", "pid": 73648, "rss_mb": 141.613, "state": "S (sleeping)", "threads": 106}, "restart_count": 397, "restart_count_per_hour": 7, "rtsp_timeout_count": 344, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 398}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.816, "sample_count": 2776, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1244616, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:31:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.521, "last_frame_pushed_at": "2026-07-01T02:31:09+09:00", "last_frame_read_age_sec": 4.608, "last_frame_read_at": "2026-07-01T02:31:09+09:00", "mjpeg": {"age_sec": 4.505, "exists": true, "mtime": "2026-07-01T02:31:09.752882+09:00", "mtime_epoch": 1782840669.7528818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 131460}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:31:14+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4958.133, "name": "celery", "pid": 267, "rss_mb": 4958.133, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 13, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.841, "pushed": 3.841, "read": 3.807}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.227, "count": 115, "max_ms": 26.744, "p95_ms": 23.22}, "gstreamer_push": {"avg_ms": 1.224, "count": 115, "max_ms": 1.433, "p95_ms": 1.332}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.623, "count": 115, "max_ms": 43.841, "p95_ms": 39.626}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.787, "count": 115, "max_ms": 69.776, "p95_ms": 68.497}, "yolo11_640_fire_11class": {"avg_ms": 55.623, "count": 115, "max_ms": 71.52, "p95_ms": 69.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.917, "count": 115, "max_ms": 78.873, "p95_ms": 76.827}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.198, "count": 115, "max_ms": 107.225, "p95_ms": 105.807}, "read_interval": {"avg_ms": 222.887, "count": 114, "max_ms": 10166.052, "p95_ms": 122.464}, "triton_wait": {"avg_ms": 60.928, "count": 115, "max_ms": 78.885, "p95_ms": 76.841}, "visualize": {"avg_ms": 3.412, "count": 115, "max_ms": 5.68, "p95_ms": 3.613}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11549}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10416.867, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55876ac6b180] error while decoding MB 40 4, bytestream -5 [in#0/rtsp @ 0x55876abe3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55876abe3e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 166, "h264_decode_error_count": 171, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:34:03+09:00", "last_error_summary": "[h264 @ 0x55876ac6b180] error while decoding MB 40 4, bytestream -5 [in#0/rtsp @ 0x55876abe3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55876abe3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:19:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 863.705, "last_started_at": "2026-07-01T02:19:39+09:00", "pid": 73496, "process": {"exists": false, "pid": 73496}, "restart_count": 165, "restart_count_per_hour": 9, "rtsp_timeout_count": 268, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 166}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.987, "sample_count": 2732, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1385965, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:34:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T02:34:02+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T02:34:02+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T02:34:02.979870+09:00", "mtime_epoch": 1782840842.9798696, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93609}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:34:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1714.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.685, "pushed": 0.685, "read": 0.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.829, "count": 12, "max_ms": 20.95, "p95_ms": 20.95}, "gstreamer_push": {"avg_ms": 1.253, "count": 12, "max_ms": 1.334, "p95_ms": 1.334}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.038, "count": 12, "max_ms": 59.6, "p95_ms": 59.6}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.57, "count": 12, "max_ms": 67.902, "p95_ms": 67.902}, "yolo11_640_fire_11class": {"avg_ms": 64.455, "count": 12, "max_ms": 68.497, "p95_ms": 68.497}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.447, "count": 12, "max_ms": 75.923, "p95_ms": 75.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.814, "count": 12, "max_ms": 103.86, "p95_ms": 103.86}, "read_interval": {"avg_ms": 2594.305, "count": 13, "max_ms": 16264.586, "p95_ms": 16264.586}, "triton_wait": {"avg_ms": 70.581, "count": 12, "max_ms": 75.934, "p95_ms": 75.934}, "visualize": {"avg_ms": 3.536, "count": 12, "max_ms": 4.217, "p95_ms": 4.217}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.526, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11549}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10416.527, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:34:03+09:00", "last_error_summary": "[h264 @ 0x55876ac6b180] error while decoding MB 40 4, bytestream -5 [in#0/rtsp @ 0x55876abe3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55876abe3e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 166, "h264_decode_error_count": 171, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:34:03+09:00", "last_error_summary": "[h264 @ 0x55876ac6b180] error while decoding MB 40 4, bytestream -5 [in#0/rtsp @ 0x55876abe3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55876abe3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:34:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 863.75, "last_started_at": "2026-07-01T02:19:39+09:00", "pid": 73496, "process": {"exists": false, "pid": 73496}, "restart_count": 165, "restart_count_per_hour": 9, "rtsp_timeout_count": 268, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 166}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.987, "sample_count": 2733, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1385966, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:34:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T02:34:03+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T02:34:02+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-01T02:34:02.979870+09:00", "mtime_epoch": 1782840842.9798696, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93609}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:34:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1714.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.74, "pushed": 0.74, "read": 0.74}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.879, "count": 13, "max_ms": 20.95, "p95_ms": 20.95}, "gstreamer_push": {"avg_ms": 1.26, "count": 13, "max_ms": 1.342, "p95_ms": 1.342}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.077, "count": 13, "max_ms": 59.6, "p95_ms": 59.6}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.935, "count": 13, "max_ms": 68.32, "p95_ms": 68.32}, "yolo11_640_fire_11class": {"avg_ms": 64.695, "count": 13, "max_ms": 68.497, "p95_ms": 68.497}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.71, "count": 13, "max_ms": 75.923, "p95_ms": 75.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.15, "count": 13, "max_ms": 103.86, "p95_ms": 103.86}, "read_interval": {"avg_ms": 2594.305, "count": 13, "max_ms": 16264.586, "p95_ms": 16264.586}, "triton_wait": {"avg_ms": 70.835, "count": 13, "max_ms": 75.934, "p95_ms": 75.934}, "visualize": {"avg_ms": 3.544, "count": 13, "max_ms": 4.217, "p95_ms": 4.217}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.571, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11549}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10416.617, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 166, "h264_decode_error_count": 171, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:34:03+09:00", "last_error_summary": "[h264 @ 0x55876ac6b180] error while decoding MB 40 4, bytestream -5 [in#0/rtsp @ 0x55876abe3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55876abe3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:34:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 863.757, "last_started_at": "2026-07-01T02:19:39+09:00", "pid": 73496, "process": {"exists": false, "pid": 73496}, "restart_count": 165, "restart_count_per_hour": 9, "rtsp_timeout_count": 268, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 166}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.987, "sample_count": 2733, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1385966, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:34:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T02:34:03+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-01T02:34:02+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-01T02:34:02.979870+09:00", "mtime_epoch": 1782840842.9798696, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93609}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:34:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1714.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.74, "pushed": 0.74, "read": 0.74}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.879, "count": 13, "max_ms": 20.95, "p95_ms": 20.95}, "gstreamer_push": {"avg_ms": 1.26, "count": 13, "max_ms": 1.342, "p95_ms": 1.342}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.077, "count": 13, "max_ms": 59.6, "p95_ms": 59.6}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.935, "count": 13, "max_ms": 68.32, "p95_ms": 68.32}, "yolo11_640_fire_11class": {"avg_ms": 64.695, "count": 13, "max_ms": 68.497, "p95_ms": 68.497}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.71, "count": 13, "max_ms": 75.923, "p95_ms": 75.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.15, "count": 13, "max_ms": 103.86, "p95_ms": 103.86}, "read_interval": {"avg_ms": 2594.305, "count": 13, "max_ms": 16264.586, "p95_ms": 16264.586}, "triton_wait": {"avg_ms": 70.835, "count": 13, "max_ms": 75.934, "p95_ms": 75.934}, "visualize": {"avg_ms": 3.544, "count": 13, "max_ms": 4.217, "p95_ms": 4.217}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.578, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11549}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10437.211, "limit_mb": 8796093022207.996}, "detail": {"pid": 73755, "restart_count": 166, "start_count": 167}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 166, "h264_decode_error_count": 171, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:34:03+09:00", "last_error_summary": "[h264 @ 0x55876ac6b180] error while decoding MB 40 4, bytestream -5 [in#0/rtsp @ 0x55876abe3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55876abe3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:34:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:34:05+09:00", "pid": 73755, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.379, "name": "ffmpeg", "pid": 73755, "rss_mb": 57.379, "state": "S (sleeping)", "threads": 1}, "restart_count": 166, "restart_count_per_hour": 10, "rtsp_timeout_count": 268, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 167}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.005, "sample_count": 2683, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1385966, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:34:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.642, "last_frame_pushed_at": "2026-07-01T02:34:03+09:00", "last_frame_read_age_sec": 4.746, "last_frame_read_at": "2026-07-01T02:34:02+09:00", "mjpeg": {"age_sec": 4.625, "exists": true, "mtime": "2026-07-01T02:34:03.095872+09:00", "mtime_epoch": 1782840843.0958717, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93882}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:34:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1714.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.585, "pushed": 0.585, "read": 0.585}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.879, "count": 13, "max_ms": 20.95, "p95_ms": 20.95}, "gstreamer_push": {"avg_ms": 1.26, "count": 13, "max_ms": 1.342, "p95_ms": 1.342}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.077, "count": 13, "max_ms": 59.6, "p95_ms": 59.6}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.935, "count": 13, "max_ms": 68.32, "p95_ms": 68.32}, "yolo11_640_fire_11class": {"avg_ms": 64.695, "count": 13, "max_ms": 68.497, "p95_ms": 68.497}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.71, "count": 13, "max_ms": 75.923, "p95_ms": 75.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.15, "count": 13, "max_ms": 103.86, "p95_ms": 103.86}, "read_interval": {"avg_ms": 2594.305, "count": 13, "max_ms": 16264.586, "p95_ms": 16264.586}, "triton_wait": {"avg_ms": 70.835, "count": 13, "max_ms": 75.934, "p95_ms": 75.934}, "visualize": {"avg_ms": 3.544, "count": 13, "max_ms": 4.217, "p95_ms": 4.217}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.208, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11555}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10420.621, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5556bd961e40] error while decoding MB 87 2, bytestream -27 [in#0/rtsp @ 0x5556bd3d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556bd3d7e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 167, "h264_decode_error_count": 172, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:36:05+09:00", "last_error_summary": "[h264 @ 0x5556bd961e40] error while decoding MB 87 2, bytestream -27 [in#0/rtsp @ 0x5556bd3d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556bd3d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:34:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.472, "last_started_at": "2026-07-01T02:34:05+09:00", "pid": 73755, "process": {"exists": false, "pid": 73755}, "restart_count": 166, "restart_count_per_hour": 10, "rtsp_timeout_count": 270, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 167}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.682, "sample_count": 2182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1386715, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:36:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-01T02:36:05+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T02:36:05+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-01T02:36:05.125996+09:00", "mtime_epoch": 1782840965.1259956, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91943}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:36:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1714.887, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.801}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.824, "count": 13, "max_ms": 22.696, "p95_ms": 22.696}, "gstreamer_push": {"avg_ms": 1.256, "count": 13, "max_ms": 1.412, "p95_ms": 1.412}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.794, "count": 13, "max_ms": 45.598, "p95_ms": 45.598}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.959, "count": 13, "max_ms": 56.194, "p95_ms": 56.194}, "yolo11_640_fire_11class": {"avg_ms": 50.33, "count": 13, "max_ms": 56.207, "p95_ms": 56.207}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.061, "count": 13, "max_ms": 54.944, "p95_ms": 54.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.567, "count": 13, "max_ms": 85.555, "p95_ms": 85.555}, "read_interval": {"avg_ms": 2477.069, "count": 14, "max_ms": 17279.54, "p95_ms": 17279.54}, "triton_wait": {"avg_ms": 50.596, "count": 13, "max_ms": 56.218, "p95_ms": 56.218}, "visualize": {"avg_ms": 3.434, "count": 13, "max_ms": 3.884, "p95_ms": 3.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.468, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11555}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10420.34, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:36:05+09:00", "last_error_summary": "[h264 @ 0x5556bd961e40] error while decoding MB 87 2, bytestream -27 [in#0/rtsp @ 0x5556bd3d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556bd3d7e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 167, "h264_decode_error_count": 172, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:36:05+09:00", "last_error_summary": "[h264 @ 0x5556bd961e40] error while decoding MB 87 2, bytestream -27 [in#0/rtsp @ 0x5556bd3d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556bd3d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:36:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.496, "last_started_at": "2026-07-01T02:34:05+09:00", "pid": 73755, "process": {"exists": false, "pid": 73755}, "restart_count": 166, "restart_count_per_hour": 10, "rtsp_timeout_count": 270, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 167}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.682, "sample_count": 2183, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1386716, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:36:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T02:36:05+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-01T02:36:05+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-01T02:36:05.125996+09:00", "mtime_epoch": 1782840965.1259956, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91943}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:36:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1714.887, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.8, "pushed": 0.8, "read": 0.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.03, "count": 14, "max_ms": 23.717, "p95_ms": 23.717}, "gstreamer_push": {"avg_ms": 1.263, "count": 14, "max_ms": 1.412, "p95_ms": 1.412}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.662, "count": 14, "max_ms": 45.598, "p95_ms": 45.598}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.82, "count": 14, "max_ms": 56.194, "p95_ms": 56.194}, "yolo11_640_fire_11class": {"avg_ms": 50.271, "count": 14, "max_ms": 56.207, "p95_ms": 56.207}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.965, "count": 14, "max_ms": 54.944, "p95_ms": 54.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.911, "count": 14, "max_ms": 85.555, "p95_ms": 85.555}, "read_interval": {"avg_ms": 2477.069, "count": 14, "max_ms": 17279.54, "p95_ms": 17279.54}, "triton_wait": {"avg_ms": 50.519, "count": 14, "max_ms": 56.218, "p95_ms": 56.218}, "visualize": {"avg_ms": 3.627, "count": 14, "max_ms": 6.137, "p95_ms": 6.137}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.492, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11555}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10420.191, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 167, "h264_decode_error_count": 172, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:36:05+09:00", "last_error_summary": "[h264 @ 0x5556bd961e40] error while decoding MB 87 2, bytestream -27 [in#0/rtsp @ 0x5556bd3d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556bd3d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:36:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.503, "last_started_at": "2026-07-01T02:34:05+09:00", "pid": 73755, "process": {"exists": false, "pid": 73755}, "restart_count": 166, "restart_count_per_hour": 10, "rtsp_timeout_count": 270, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 167}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.682, "sample_count": 2183, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1386716, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:36:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T02:36:05+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-01T02:36:05+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-01T02:36:05.125996+09:00", "mtime_epoch": 1782840965.1259956, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91943}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:36:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1714.887, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.8, "pushed": 0.8, "read": 0.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.03, "count": 14, "max_ms": 23.717, "p95_ms": 23.717}, "gstreamer_push": {"avg_ms": 1.263, "count": 14, "max_ms": 1.412, "p95_ms": 1.412}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.662, "count": 14, "max_ms": 45.598, "p95_ms": 45.598}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.82, "count": 14, "max_ms": 56.194, "p95_ms": 56.194}, "yolo11_640_fire_11class": {"avg_ms": 50.271, "count": 14, "max_ms": 56.207, "p95_ms": 56.207}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.965, "count": 14, "max_ms": 54.944, "p95_ms": 54.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.911, "count": 14, "max_ms": 85.555, "p95_ms": 85.555}, "read_interval": {"avg_ms": 2477.069, "count": 14, "max_ms": 17279.54, "p95_ms": 17279.54}, "triton_wait": {"avg_ms": 50.519, "count": 14, "max_ms": 56.218, "p95_ms": 56.218}, "visualize": {"avg_ms": 3.627, "count": 14, "max_ms": 6.137, "p95_ms": 6.137}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.499, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11555}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10533.484, "limit_mb": 8796093022207.996}, "detail": {"pid": 73907, "restart_count": 167, "start_count": 168}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 167, "h264_decode_error_count": 172, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:36:05+09:00", "last_error_summary": "[h264 @ 0x5556bd961e40] error while decoding MB 87 2, bytestream -27 [in#0/rtsp @ 0x5556bd3d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5556bd3d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:36:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:36:07+09:00", "pid": 73907, "process": {"exists": true, "fd_count": 5, "hwm_mb": 145.742, "name": "ffmpeg", "pid": 73907, "rss_mb": 145.742, "state": "S (sleeping)", "threads": 44}, "restart_count": 167, "restart_count_per_hour": 11, "rtsp_timeout_count": 270, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 168}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.75, "sample_count": 2131, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1386716, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:36:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.783, "last_frame_pushed_at": "2026-07-01T02:36:05+09:00", "last_frame_read_age_sec": 4.87, "last_frame_read_at": "2026-07-01T02:36:05+09:00", "mjpeg": {"age_sec": 4.768, "exists": true, "mtime": "2026-07-01T02:36:05.225997+09:00", "mtime_epoch": 1782840965.2259972, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90153}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:36:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1714.887, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.629, "pushed": 0.629, "read": 0.629}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.03, "count": 14, "max_ms": 23.717, "p95_ms": 23.717}, "gstreamer_push": {"avg_ms": 1.263, "count": 14, "max_ms": 1.412, "p95_ms": 1.412}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.662, "count": 14, "max_ms": 45.598, "p95_ms": 45.598}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.82, "count": 14, "max_ms": 56.194, "p95_ms": 56.194}, "yolo11_640_fire_11class": {"avg_ms": 50.271, "count": 14, "max_ms": 56.207, "p95_ms": 56.207}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.965, "count": 14, "max_ms": 54.944, "p95_ms": 54.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.911, "count": 14, "max_ms": 85.555, "p95_ms": 85.555}, "read_interval": {"avg_ms": 2477.069, "count": 14, "max_ms": 17279.54, "p95_ms": 17279.54}, "triton_wait": {"avg_ms": 50.519, "count": 14, "max_ms": 56.218, "p95_ms": 56.218}, "visualize": {"avg_ms": 3.627, "count": 14, "max_ms": 6.137, "p95_ms": 6.137}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.269, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11573}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10446.77, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5585ea18ac80] error while decoding MB 66 22, bytestream -29 [in#0/rtsp @ 0x5585e9990e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585e9990e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 168, "h264_decode_error_count": 173, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:40:48+09:00", "last_error_summary": "[h264 @ 0x5585ea18ac80] error while decoding MB 66 22, bytestream -29 [in#0/rtsp @ 0x5585e9990e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585e9990e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:36:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 280.776, "last_started_at": "2026-07-01T02:36:07+09:00", "pid": 73907, "process": {"exists": false, "pid": 73907}, "restart_count": 167, "restart_count_per_hour": 10, "rtsp_timeout_count": 272, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 168}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.142, "sample_count": 2151, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1388854, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:40:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-01T02:40:48+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T02:40:48+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-01T02:40:48.702974+09:00", "mtime_epoch": 1782841248.7029736, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91358}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:40:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1715.918, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.69, "pushed": 0.69, "read": 0.747}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.092, "count": 12, "max_ms": 21.532, "p95_ms": 21.532}, "gstreamer_push": {"avg_ms": 1.266, "count": 12, "max_ms": 1.403, "p95_ms": 1.403}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.881, "count": 12, "max_ms": 48.286, "p95_ms": 48.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.899, "count": 12, "max_ms": 54.022, "p95_ms": 54.022}, "yolo11_640_fire_11class": {"avg_ms": 51.903, "count": 12, "max_ms": 56.312, "p95_ms": 56.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.031, "count": 12, "max_ms": 54.557, "p95_ms": 54.557}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.318, "count": 12, "max_ms": 84.605, "p95_ms": 84.605}, "read_interval": {"avg_ms": 2651.859, "count": 13, "max_ms": 17145.1, "p95_ms": 17145.1}, "triton_wait": {"avg_ms": 51.916, "count": 12, "max_ms": 56.324, "p95_ms": 56.324}, "visualize": {"avg_ms": 3.452, "count": 12, "max_ms": 3.559, "p95_ms": 3.559}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.4, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11573}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10446.562, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:40:48+09:00", "last_error_summary": "[h264 @ 0x5585ea18ac80] error while decoding MB 66 22, bytestream -29 [in#0/rtsp @ 0x5585e9990e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585e9990e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 168, "h264_decode_error_count": 173, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:40:48+09:00", "last_error_summary": "[h264 @ 0x5585ea18ac80] error while decoding MB 66 22, bytestream -29 [in#0/rtsp @ 0x5585e9990e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585e9990e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:40:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 280.803, "last_started_at": "2026-07-01T02:36:07+09:00", "pid": 73907, "process": {"exists": false, "pid": 73907}, "restart_count": 167, "restart_count_per_hour": 10, "rtsp_timeout_count": 272, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 168}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.142, "sample_count": 2152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1388855, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:40:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T02:40:48+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-01T02:40:48+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-01T02:40:48.702974+09:00", "mtime_epoch": 1782841248.7029736, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91358}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:40:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1715.918, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.746, "pushed": 0.746, "read": 0.746}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.393, "count": 13, "max_ms": 24.007, "p95_ms": 24.007}, "gstreamer_push": {"avg_ms": 1.269, "count": 13, "max_ms": 1.403, "p95_ms": 1.403}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.731, "count": 13, "max_ms": 48.286, "p95_ms": 48.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.675, "count": 13, "max_ms": 54.022, "p95_ms": 54.022}, "yolo11_640_fire_11class": {"avg_ms": 51.811, "count": 13, "max_ms": 56.312, "p95_ms": 56.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.832, "count": 13, "max_ms": 54.557, "p95_ms": 54.557}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.733, "count": 13, "max_ms": 85.724, "p95_ms": 85.724}, "read_interval": {"avg_ms": 2651.859, "count": 13, "max_ms": 17145.1, "p95_ms": 17145.1}, "triton_wait": {"avg_ms": 51.824, "count": 13, "max_ms": 56.324, "p95_ms": 56.324}, "visualize": {"avg_ms": 3.59, "count": 13, "max_ms": 5.24, "p95_ms": 5.24}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.425, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11573}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10446.418, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 168, "h264_decode_error_count": 173, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:40:48+09:00", "last_error_summary": "[h264 @ 0x5585ea18ac80] error while decoding MB 66 22, bytestream -29 [in#0/rtsp @ 0x5585e9990e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585e9990e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:40:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 280.809, "last_started_at": "2026-07-01T02:36:07+09:00", "pid": 73907, "process": {"exists": false, "pid": 73907}, "restart_count": 167, "restart_count_per_hour": 10, "rtsp_timeout_count": 272, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 168}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.142, "sample_count": 2152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1388855, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:40:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T02:40:48+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-01T02:40:48+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-01T02:40:48.702974+09:00", "mtime_epoch": 1782841248.7029736, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91358}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:40:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1715.918, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.746, "pushed": 0.746, "read": 0.746}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.393, "count": 13, "max_ms": 24.007, "p95_ms": 24.007}, "gstreamer_push": {"avg_ms": 1.269, "count": 13, "max_ms": 1.403, "p95_ms": 1.403}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.731, "count": 13, "max_ms": 48.286, "p95_ms": 48.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.675, "count": 13, "max_ms": 54.022, "p95_ms": 54.022}, "yolo11_640_fire_11class": {"avg_ms": 51.811, "count": 13, "max_ms": 56.312, "p95_ms": 56.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.832, "count": 13, "max_ms": 54.557, "p95_ms": 54.557}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.733, "count": 13, "max_ms": 85.724, "p95_ms": 85.724}, "read_interval": {"avg_ms": 2651.859, "count": 13, "max_ms": 17145.1, "p95_ms": 17145.1}, "triton_wait": {"avg_ms": 51.824, "count": 13, "max_ms": 56.324, "p95_ms": 56.324}, "visualize": {"avg_ms": 3.59, "count": 13, "max_ms": 5.24, "p95_ms": 5.24}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.432, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11573}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10624.059, "limit_mb": 8796093022207.996}, "detail": {"pid": 74059, "restart_count": 168, "start_count": 169}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 168, "h264_decode_error_count": 173, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:40:48+09:00", "last_error_summary": "[h264 @ 0x5585ea18ac80] error while decoding MB 66 22, bytestream -29 [in#0/rtsp @ 0x5585e9990e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585e9990e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:40:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:40:51+09:00", "pid": 74059, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.078, "name": "ffmpeg", "pid": 74059, "rss_mb": 204.078, "state": "S (sleeping)", "threads": 106}, "restart_count": 168, "restart_count_per_hour": 11, "rtsp_timeout_count": 272, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 169}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.142, "sample_count": 2152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1388855, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:40:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.488, "last_frame_pushed_at": "2026-07-01T02:40:48+09:00", "last_frame_read_age_sec": 4.575, "last_frame_read_at": "2026-07-01T02:40:48+09:00", "mjpeg": {"age_sec": 4.472, "exists": true, "mtime": "2026-07-01T02:40:48.802975+09:00", "mtime_epoch": 1782841248.8029754, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91985}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:40:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1715.918, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.593, "pushed": 0.593, "read": 0.593}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.393, "count": 13, "max_ms": 24.007, "p95_ms": 24.007}, "gstreamer_push": {"avg_ms": 1.269, "count": 13, "max_ms": 1.403, "p95_ms": 1.403}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.731, "count": 13, "max_ms": 48.286, "p95_ms": 48.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.675, "count": 13, "max_ms": 54.022, "p95_ms": 54.022}, "yolo11_640_fire_11class": {"avg_ms": 51.811, "count": 13, "max_ms": 56.312, "p95_ms": 56.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.832, "count": 13, "max_ms": 54.557, "p95_ms": 54.557}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.733, "count": 13, "max_ms": 85.724, "p95_ms": 85.724}, "read_interval": {"avg_ms": 2651.859, "count": 13, "max_ms": 17145.1, "p95_ms": 17145.1}, "triton_wait": {"avg_ms": 51.824, "count": 13, "max_ms": 56.324, "p95_ms": 56.324}, "visualize": {"avg_ms": 3.59, "count": 13, "max_ms": 5.24, "p95_ms": 5.24}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.905, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10419}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10473.223, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x558315075940] error while decoding MB 42 34, bytestream -5 [h264 @ 0x5583150a2500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5583150a2500] error while decoding MB 0 11, bytestream 6374 [h264 @ 0x558315061840] cabac decode of qscale diff failed at 16 28 [h264 @ 0x558315061840] error while decoding MB 16 28, bytestream 8102 [h264 @ 0x5583150a2500] top block unavailable for requested intra mode [h264 @ 0x5583150a2500] error while decoding MB 40 0, bytestream 2611 [in", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 364, "h264_decode_error_count": 1191, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:41:40+09:00", "last_error_summary": "[h264 @ 0x558315075940] error while decoding MB 42 34, bytestream -5 [h264 @ 0x5583150a2500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5583150a2500] error while decoding MB 0 11, bytestream 6374 [h264 @ 0x558315061840] cabac decode of qscale diff failed at 16 28 [h264 @ 0x558315061840] error while decoding MB 16 28, bytestream 8102 [h264 @ 0x5583150a2500] top block unavailable for requested intra mode [h264 @ 0x5583150a2500] error while decoding MB 40 0, bytestream 2611 [in", "last_exit_at": "2026-07-01T02:31:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 627.824, "last_started_at": "2026-07-01T02:31:12+09:00", "pid": 73648, "process": {"exists": false, "pid": 73648}, "restart_count": 397, "restart_count_per_hour": 6, "rtsp_timeout_count": 346, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 398}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.192, "sample_count": 2545, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1250364, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:41:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T02:41:40+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T02:41:40+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T02:41:40.019879+09:00", "mtime_epoch": 1782841300.019879, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118241}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:41:40+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4960.969, "name": "celery", "pid": 267, "rss_mb": 4960.969, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.238, "pushed": 3.238, "read": 3.238}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.524, "count": 97, "max_ms": 23.608, "p95_ms": 22.519}, "gstreamer_push": {"avg_ms": 1.206, "count": 97, "max_ms": 1.366, "p95_ms": 1.316}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.983, "count": 97, "max_ms": 45.017, "p95_ms": 42.167}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.168, "count": 97, "max_ms": 69.614, "p95_ms": 66.991}, "yolo11_640_fire_11class": {"avg_ms": 62.509, "count": 97, "max_ms": 71.85, "p95_ms": 68.321}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.1, "count": 97, "max_ms": 78.564, "p95_ms": 77.29}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.67, "count": 97, "max_ms": 107.033, "p95_ms": 105.422}, "read_interval": {"avg_ms": 309.119, "count": 97, "max_ms": 19068.042, "p95_ms": 120.375}, "triton_wait": {"avg_ms": 70.112, "count": 97, "max_ms": 78.577, "p95_ms": 77.302}, "visualize": {"avg_ms": 3.365, "count": 97, "max_ms": 3.92, "p95_ms": 3.55}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10419}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10472.387, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:41:40+09:00", "last_error_summary": "[h264 @ 0x558315075940] error while decoding MB 42 34, bytestream -5 [h264 @ 0x5583150a2500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5583150a2500] error while decoding MB 0 11, bytestream 6374 [h264 @ 0x558315061840] cabac decode of qscale diff failed at 16 28 [h264 @ 0x558315061840] error while decoding MB 16 28, bytestream 8102 [h264 @ 0x5583150a2500] top block unavailable for requested intra mode [h264 @ 0x5583150a2500] error while decoding MB 40 0, bytestream 2611 [in", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 364, "h264_decode_error_count": 1191, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:41:40+09:00", "last_error_summary": "[h264 @ 0x558315075940] error while decoding MB 42 34, bytestream -5 [h264 @ 0x5583150a2500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5583150a2500] error while decoding MB 0 11, bytestream 6374 [h264 @ 0x558315061840] cabac decode of qscale diff failed at 16 28 [h264 @ 0x558315061840] error while decoding MB 16 28, bytestream 8102 [h264 @ 0x5583150a2500] top block unavailable for requested intra mode [h264 @ 0x5583150a2500] error while decoding MB 40 0, bytestream 2611 [in", "last_exit_at": "2026-07-01T02:41:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 627.868, "last_started_at": "2026-07-01T02:31:12+09:00", "pid": 73648, "process": {"exists": false, "pid": 73648}, "restart_count": 397, "restart_count_per_hour": 6, "rtsp_timeout_count": 346, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 398}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.192, "sample_count": 2546, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1250365, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:41:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T02:41:40+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-01T02:41:40+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-01T02:41:40.019879+09:00", "mtime_epoch": 1782841300.019879, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118241}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:41:40+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4960.969, "name": "celery", "pid": 267, "rss_mb": 4960.969, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.267, "pushed": 3.267, "read": 3.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.527, "count": 97, "max_ms": 23.608, "p95_ms": 22.519}, "gstreamer_push": {"avg_ms": 1.207, "count": 98, "max_ms": 1.366, "p95_ms": 1.316}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.875, "count": 97, "max_ms": 45.017, "p95_ms": 42.167}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.198, "count": 97, "max_ms": 69.614, "p95_ms": 66.991}, "yolo11_640_fire_11class": {"avg_ms": 62.557, "count": 97, "max_ms": 71.85, "p95_ms": 68.321}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.159, "count": 97, "max_ms": 78.564, "p95_ms": 77.29}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.73, "count": 98, "max_ms": 107.033, "p95_ms": 105.422}, "read_interval": {"avg_ms": 309.119, "count": 97, "max_ms": 19068.042, "p95_ms": 120.375}, "triton_wait": {"avg_ms": 70.171, "count": 97, "max_ms": 78.577, "p95_ms": 77.302}, "visualize": {"avg_ms": 3.366, "count": 97, "max_ms": 3.92, "p95_ms": 3.55}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10419}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10472.273, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 364, "h264_decode_error_count": 1191, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:41:40+09:00", "last_error_summary": "[h264 @ 0x558315075940] error while decoding MB 42 34, bytestream -5 [h264 @ 0x5583150a2500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5583150a2500] error while decoding MB 0 11, bytestream 6374 [h264 @ 0x558315061840] cabac decode of qscale diff failed at 16 28 [h264 @ 0x558315061840] error while decoding MB 16 28, bytestream 8102 [h264 @ 0x5583150a2500] top block unavailable for requested intra mode [h264 @ 0x5583150a2500] error while decoding MB 40 0, bytestream 2611 [in", "last_exit_at": "2026-07-01T02:41:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 627.876, "last_started_at": "2026-07-01T02:31:12+09:00", "pid": 73648, "process": {"exists": false, "pid": 73648}, "restart_count": 397, "restart_count_per_hour": 6, "rtsp_timeout_count": 346, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 398}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.192, "sample_count": 2546, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1250365, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:41:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T02:41:40+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-01T02:41:40+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-01T02:41:40.019879+09:00", "mtime_epoch": 1782841300.019879, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118241}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:41:40+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4960.969, "name": "celery", "pid": 267, "rss_mb": 4960.969, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.234, "pushed": 3.234, "read": 3.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.527, "count": 97, "max_ms": 23.608, "p95_ms": 22.519}, "gstreamer_push": {"avg_ms": 1.207, "count": 97, "max_ms": 1.366, "p95_ms": 1.316}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.875, "count": 97, "max_ms": 45.017, "p95_ms": 42.167}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.198, "count": 97, "max_ms": 69.614, "p95_ms": 66.991}, "yolo11_640_fire_11class": {"avg_ms": 62.557, "count": 97, "max_ms": 71.85, "p95_ms": 68.321}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.159, "count": 97, "max_ms": 78.564, "p95_ms": 77.29}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.732, "count": 97, "max_ms": 107.033, "p95_ms": 105.422}, "read_interval": {"avg_ms": 309.119, "count": 97, "max_ms": 19068.042, "p95_ms": 120.375}, "triton_wait": {"avg_ms": 70.171, "count": 97, "max_ms": 78.577, "p95_ms": 77.302}, "visualize": {"avg_ms": 3.366, "count": 97, "max_ms": 3.92, "p95_ms": 3.55}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10419}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10583.684, "limit_mb": 8796093022207.996}, "detail": {"pid": 74211, "restart_count": 398, "start_count": 399}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 364, "h264_decode_error_count": 1191, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:41:40+09:00", "last_error_summary": "[h264 @ 0x558315075940] error while decoding MB 42 34, bytestream -5 [h264 @ 0x5583150a2500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5583150a2500] error while decoding MB 0 11, bytestream 6374 [h264 @ 0x558315061840] cabac decode of qscale diff failed at 16 28 [h264 @ 0x558315061840] error while decoding MB 16 28, bytestream 8102 [h264 @ 0x5583150a2500] top block unavailable for requested intra mode [h264 @ 0x5583150a2500] error while decoding MB 40 0, bytestream 2611 [in", "last_exit_at": "2026-07-01T02:41:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:41:42+09:00", "pid": 74211, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.812, "name": "ffmpeg", "pid": 74211, "rss_mb": 141.812, "state": "S (sleeping)", "threads": 106}, "restart_count": 398, "restart_count_per_hour": 7, "rtsp_timeout_count": 346, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 399}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.208, "sample_count": 2506, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1250365, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:41:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.607, "last_frame_pushed_at": "2026-07-01T02:41:40+09:00", "last_frame_read_age_sec": 4.713, "last_frame_read_at": "2026-07-01T02:41:40+09:00", "mjpeg": {"age_sec": 4.591, "exists": true, "mtime": "2026-07-01T02:41:40.135881+09:00", "mtime_epoch": 1782841300.1358812, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114725}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:41:44+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4960.969, "name": "celery", "pid": 267, "rss_mb": 4960.969, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 57, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.9, "pushed": 1.9, "read": 1.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.677, "count": 57, "max_ms": 23.608, "p95_ms": 23.267}, "gstreamer_push": {"avg_ms": 1.218, "count": 57, "max_ms": 1.366, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.5, "count": 57, "max_ms": 42.627, "p95_ms": 41.129}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.883, "count": 57, "max_ms": 69.614, "p95_ms": 68.468}, "yolo11_640_fire_11class": {"avg_ms": 62.373, "count": 57, "max_ms": 71.85, "p95_ms": 69.541}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.737, "count": 57, "max_ms": 78.564, "p95_ms": 77.376}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.45, "count": 57, "max_ms": 107.033, "p95_ms": 105.843}, "read_interval": {"avg_ms": 445.589, "count": 57, "max_ms": 19068.042, "p95_ms": 120.439}, "triton_wait": {"avg_ms": 69.749, "count": 57, "max_ms": 78.577, "p95_ms": 77.388}, "visualize": {"avg_ms": 3.378, "count": 57, "max_ms": 3.92, "p95_ms": 3.579}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10424.438, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e1f5bae0c0] error while decoding MB 74 13, bytestream -7 [in#0/rtsp @ 0x55e1f507ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1f507ae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 169, "h264_decode_error_count": 174, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:44:16+09:00", "last_error_summary": "[h264 @ 0x55e1f5bae0c0] error while decoding MB 74 13, bytestream -7 [in#0/rtsp @ 0x55e1f507ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1f507ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:40:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 205.118, "last_started_at": "2026-07-01T02:40:51+09:00", "pid": 74059, "process": {"exists": false, "pid": 74059}, "restart_count": 168, "restart_count_per_hour": 10, "rtsp_timeout_count": 274, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 169}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.74, "sample_count": 2129, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1390469, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:44:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-01T02:44:16+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-01T02:44:16+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-01T02:44:16.334644+09:00", "mtime_epoch": 1782841456.3346436, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 88553}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:44:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1716.691, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.696, "pushed": 0.696, "read": 0.754}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.339, "count": 12, "max_ms": 22.255, "p95_ms": 22.255}, "gstreamer_push": {"avg_ms": 1.273, "count": 12, "max_ms": 1.643, "p95_ms": 1.643}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.721, "count": 12, "max_ms": 61.175, "p95_ms": 61.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.816, "count": 12, "max_ms": 64.393, "p95_ms": 64.393}, "yolo11_640_fire_11class": {"avg_ms": 51.924, "count": 12, "max_ms": 66.69, "p95_ms": 66.69}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.457, "count": 12, "max_ms": 73.228, "p95_ms": 73.228}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.328, "count": 12, "max_ms": 100.692, "p95_ms": 100.692}, "read_interval": {"avg_ms": 2587.826, "count": 13, "max_ms": 16466.223, "p95_ms": 16466.223}, "triton_wait": {"avg_ms": 52.705, "count": 12, "max_ms": 73.24, "p95_ms": 73.24}, "visualize": {"avg_ms": 3.462, "count": 12, "max_ms": 3.603, "p95_ms": 3.603}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.237, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10424.273, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:44:16+09:00", "last_error_summary": "[h264 @ 0x55e1f5bae0c0] error while decoding MB 74 13, bytestream -7 [in#0/rtsp @ 0x55e1f507ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1f507ae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 169, "h264_decode_error_count": 174, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:44:16+09:00", "last_error_summary": "[h264 @ 0x55e1f5bae0c0] error while decoding MB 74 13, bytestream -7 [in#0/rtsp @ 0x55e1f507ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1f507ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:44:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 205.14, "last_started_at": "2026-07-01T02:40:51+09:00", "pid": 74059, "process": {"exists": false, "pid": 74059}, "restart_count": 168, "restart_count_per_hour": 10, "rtsp_timeout_count": 274, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 169}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.74, "sample_count": 2129, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1390470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:44:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T02:44:16+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-01T02:44:16+09:00", "mjpeg": {"age_sec": 0.079, "exists": true, "mtime": "2026-07-01T02:44:16.334644+09:00", "mtime_epoch": 1782841456.3346436, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 88553}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:44:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1716.691, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.753, "pushed": 0.753, "read": 0.753}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.31, "count": 13, "max_ms": 22.255, "p95_ms": 22.255}, "gstreamer_push": {"avg_ms": 1.261, "count": 13, "max_ms": 1.643, "p95_ms": 1.643}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.149, "count": 13, "max_ms": 61.175, "p95_ms": 61.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.371, "count": 13, "max_ms": 64.393, "p95_ms": 64.393}, "yolo11_640_fire_11class": {"avg_ms": 51.38, "count": 13, "max_ms": 66.69, "p95_ms": 66.69}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.989, "count": 13, "max_ms": 73.228, "p95_ms": 73.228}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.812, "count": 13, "max_ms": 100.692, "p95_ms": 100.692}, "read_interval": {"avg_ms": 2587.826, "count": 13, "max_ms": 16466.223, "p95_ms": 16466.223}, "triton_wait": {"avg_ms": 52.142, "count": 13, "max_ms": 73.24, "p95_ms": 73.24}, "visualize": {"avg_ms": 3.539, "count": 13, "max_ms": 4.467, "p95_ms": 4.467}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.26, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10424.266, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 169, "h264_decode_error_count": 174, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:44:16+09:00", "last_error_summary": "[h264 @ 0x55e1f5bae0c0] error while decoding MB 74 13, bytestream -7 [in#0/rtsp @ 0x55e1f507ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1f507ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:44:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 205.147, "last_started_at": "2026-07-01T02:40:51+09:00", "pid": 74059, "process": {"exists": false, "pid": 74059}, "restart_count": 168, "restart_count_per_hour": 10, "rtsp_timeout_count": 274, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 169}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.74, "sample_count": 2129, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1390470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:44:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T02:44:16+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-01T02:44:16+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-01T02:44:16.334644+09:00", "mtime_epoch": 1782841456.3346436, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 88553}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:44:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1716.691, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.753, "pushed": 0.753, "read": 0.753}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.31, "count": 13, "max_ms": 22.255, "p95_ms": 22.255}, "gstreamer_push": {"avg_ms": 1.261, "count": 13, "max_ms": 1.643, "p95_ms": 1.643}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.149, "count": 13, "max_ms": 61.175, "p95_ms": 61.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.371, "count": 13, "max_ms": 64.393, "p95_ms": 64.393}, "yolo11_640_fire_11class": {"avg_ms": 51.38, "count": 13, "max_ms": 66.69, "p95_ms": 66.69}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.989, "count": 13, "max_ms": 73.228, "p95_ms": 73.228}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.812, "count": 13, "max_ms": 100.692, "p95_ms": 100.692}, "read_interval": {"avg_ms": 2587.826, "count": 13, "max_ms": 16466.223, "p95_ms": 16466.223}, "triton_wait": {"avg_ms": 52.142, "count": 13, "max_ms": 73.24, "p95_ms": 73.24}, "visualize": {"avg_ms": 3.539, "count": 13, "max_ms": 4.467, "p95_ms": 4.467}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.266, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10547.691, "limit_mb": 8796093022207.996}, "detail": {"pid": 74318, "restart_count": 169, "start_count": 170}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 169, "h264_decode_error_count": 174, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:44:16+09:00", "last_error_summary": "[h264 @ 0x55e1f5bae0c0] error while decoding MB 74 13, bytestream -7 [in#0/rtsp @ 0x55e1f507ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1f507ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:44:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:44:19+09:00", "pid": 74318, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.246, "name": "ffmpeg", "pid": 74318, "rss_mb": 156.246, "state": "R (running)", "threads": 49}, "restart_count": 169, "restart_count_per_hour": 11, "rtsp_timeout_count": 274, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 170}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.702, "sample_count": 2089, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1390470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:44:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.661, "last_frame_pushed_at": "2026-07-01T02:44:16+09:00", "last_frame_read_age_sec": 4.737, "last_frame_read_at": "2026-07-01T02:44:16+09:00", "mjpeg": {"age_sec": 4.644, "exists": true, "mtime": "2026-07-01T02:44:16.422645+09:00", "mtime_epoch": 1782841456.4226453, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89882}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:44:21+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1716.691, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.593, "pushed": 0.593, "read": 0.593}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.31, "count": 13, "max_ms": 22.255, "p95_ms": 22.255}, "gstreamer_push": {"avg_ms": 1.261, "count": 13, "max_ms": 1.643, "p95_ms": 1.643}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.149, "count": 13, "max_ms": 61.175, "p95_ms": 61.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.371, "count": 13, "max_ms": 64.393, "p95_ms": 64.393}, "yolo11_640_fire_11class": {"avg_ms": 51.38, "count": 13, "max_ms": 66.69, "p95_ms": 66.69}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.989, "count": 13, "max_ms": 73.228, "p95_ms": 73.228}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.812, "count": 13, "max_ms": 100.692, "p95_ms": 100.692}, "read_interval": {"avg_ms": 2587.826, "count": 13, "max_ms": 16466.223, "p95_ms": 16466.223}, "triton_wait": {"avg_ms": 52.142, "count": 13, "max_ms": 73.24, "p95_ms": 73.24}, "visualize": {"avg_ms": 3.539, "count": 13, "max_ms": 4.467, "p95_ms": 4.467}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.913, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10427.293, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55c7f17eb200] missing picture in access unit with size 18 [h264 @ 0x55c7f2583200] no frame! [in#0/rtsp @ 0x55c7f17e6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c7f17e6e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55c7f17eb840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 170, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:48:48+09:00", "last_error_summary": "[NULL @ 0x55c7f17eb200] missing picture in access unit with size 18 [h264 @ 0x55c7f2583200] no frame! [in#0/rtsp @ 0x55c7f17e6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c7f17e6e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55c7f17eb840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:44:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 269.436, "last_started_at": "2026-07-01T02:44:19+09:00", "pid": 74318, "process": {"exists": false, "pid": 74318}, "restart_count": 169, "restart_count_per_hour": 11, "rtsp_timeout_count": 276, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 170}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.97, "sample_count": 2158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1392615, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:48:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.026, "last_frame_pushed_at": "2026-07-01T02:48:48+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T02:48:48+09:00", "mjpeg": {"age_sec": 0.014, "exists": true, "mtime": "2026-07-01T02:48:48.491474+09:00", "mtime_epoch": 1782841728.491474, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 88143}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:48:48+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1717.723, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.821, "pushed": 10.821, "read": 10.821}, "hints": ["ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 21.379, "count": 120, "max_ms": 26.16, "p95_ms": 24.902}, "gstreamer_push": {"avg_ms": 1.249, "count": 120, "max_ms": 1.361, "p95_ms": 1.334}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.806, "count": 120, "max_ms": 56.065, "p95_ms": 41.775}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.766, "count": 120, "max_ms": 57.175, "p95_ms": 49.515}, "yolo11_640_fire_11class": {"avg_ms": 49.406, "count": 120, "max_ms": 56.048, "p95_ms": 51.843}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.867, "count": 120, "max_ms": 57.186, "p95_ms": 49.984}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.169, "count": 120, "max_ms": 85.029, "p95_ms": 83.396}, "read_interval": {"avg_ms": 316.531, "count": 120, "max_ms": 27002.62, "p95_ms": 96.256}, "triton_wait": {"avg_ms": 49.47, "count": 120, "max_ms": 57.196, "p95_ms": 51.855}, "visualize": {"avg_ms": 3.537, "count": 120, "max_ms": 4.181, "p95_ms": 3.962}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.089, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10427.16, "limit_mb": 8796093022207.996}, "detail": {"actual_bytes": 0, "expected_bytes": 2764800}, "event_type": "invalid_frame_read_size", "ffmpeg": {"error_count": 170, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:48:48+09:00", "last_error_summary": "[NULL @ 0x55c7f17eb200] missing picture in access unit with size 18 [h264 @ 0x55c7f2583200] no frame! [in#0/rtsp @ 0x55c7f17e6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c7f17e6e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55c7f17eb840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:44:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 269.436, "last_started_at": "2026-07-01T02:44:19+09:00", "pid": 74318, "process": {"exists": false, "pid": 74318}, "restart_count": 169, "restart_count_per_hour": 11, "rtsp_timeout_count": 276, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 170}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.97, "sample_count": 2158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1392615, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:48:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.025, "last_frame_pushed_at": "2026-07-01T02:48:48+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T02:48:48+09:00", "mjpeg": {"age_sec": 0.01, "exists": true, "mtime": "2026-07-01T02:48:48.491474+09:00", "mtime_epoch": 1782841728.491474, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 88143}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:48:48+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1717.723, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.825, "pushed": 10.825, "read": 10.825}, "hints": ["ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 21.379, "count": 120, "max_ms": 26.16, "p95_ms": 24.902}, "gstreamer_push": {"avg_ms": 1.249, "count": 120, "max_ms": 1.361, "p95_ms": 1.334}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.806, "count": 120, "max_ms": 56.065, "p95_ms": 41.775}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.766, "count": 120, "max_ms": 57.175, "p95_ms": 49.515}, "yolo11_640_fire_11class": {"avg_ms": 49.406, "count": 120, "max_ms": 56.048, "p95_ms": 51.843}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.867, "count": 120, "max_ms": 57.186, "p95_ms": 49.984}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.169, "count": 120, "max_ms": 85.029, "p95_ms": 83.396}, "read_interval": {"avg_ms": 316.531, "count": 120, "max_ms": 27002.62, "p95_ms": 96.256}, "triton_wait": {"avg_ms": 49.47, "count": 120, "max_ms": 57.196, "p95_ms": 51.855}, "visualize": {"avg_ms": 3.537, "count": 120, "max_ms": 4.181, "p95_ms": 3.962}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.086, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10427.117, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 170, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:48:48+09:00", "last_error_summary": "[NULL @ 0x55c7f17eb200] missing picture in access unit with size 18 [h264 @ 0x55c7f2583200] no frame! [in#0/rtsp @ 0x55c7f17e6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c7f17e6e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55c7f17eb840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:44:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 269.454, "last_started_at": "2026-07-01T02:44:19+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 169, "restart_count_per_hour": 11, "rtsp_timeout_count": 276, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 170}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.97, "sample_count": 2158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1392615, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:48:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.044, "last_frame_pushed_at": "2026-07-01T02:48:48+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-01T02:48:48+09:00", "mjpeg": {"age_sec": 0.029, "exists": true, "mtime": "2026-07-01T02:48:48.491474+09:00", "mtime_epoch": 1782841728.491474, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 88143}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:48:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1717.723, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.807, "pushed": 10.807, "read": 10.807}, "hints": ["ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 21.379, "count": 120, "max_ms": 26.16, "p95_ms": 24.902}, "gstreamer_push": {"avg_ms": 1.249, "count": 120, "max_ms": 1.361, "p95_ms": 1.334}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.806, "count": 120, "max_ms": 56.065, "p95_ms": 41.775}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.766, "count": 120, "max_ms": 57.175, "p95_ms": 49.515}, "yolo11_640_fire_11class": {"avg_ms": 49.406, "count": 120, "max_ms": 56.048, "p95_ms": 51.843}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.867, "count": 120, "max_ms": 57.186, "p95_ms": 49.984}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.169, "count": 120, "max_ms": 85.029, "p95_ms": 83.396}, "read_interval": {"avg_ms": 316.531, "count": 120, "max_ms": 27002.62, "p95_ms": 96.256}, "triton_wait": {"avg_ms": 49.47, "count": 120, "max_ms": 57.196, "p95_ms": 51.855}, "visualize": {"avg_ms": 3.537, "count": 120, "max_ms": 4.181, "p95_ms": 3.962}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.104, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10603.578, "limit_mb": 8796093022207.996}, "detail": {"pid": 74470, "restart_count": 170, "start_count": 171}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 170, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:48:48+09:00", "last_error_summary": "[NULL @ 0x55c7f17eb200] missing picture in access unit with size 18 [h264 @ 0x55c7f2583200] no frame! [in#0/rtsp @ 0x55c7f17e6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c7f17e6e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55c7f17eb840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T02:44:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:48:50+09:00", "pid": 74470, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.293, "name": "ffmpeg", "pid": 74470, "rss_mb": 206.293, "state": "S (sleeping)", "threads": 106}, "restart_count": 170, "restart_count_per_hour": 12, "rtsp_timeout_count": 276, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 171}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.97, "sample_count": 2158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1392615, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:48:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.076, "last_frame_pushed_at": "2026-07-01T02:48:48+09:00", "last_frame_read_age_sec": 4.154, "last_frame_read_at": "2026-07-01T02:48:48+09:00", "mjpeg": {"age_sec": 4.061, "exists": true, "mtime": "2026-07-01T02:48:48.491474+09:00", "mtime_epoch": 1782841728.491474, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 88143}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T02:48:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1717.723, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.928, "pushed": 7.928, "read": 7.928}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 21.379, "count": 120, "max_ms": 26.16, "p95_ms": 24.902}, "gstreamer_push": {"avg_ms": 1.249, "count": 120, "max_ms": 1.361, "p95_ms": 1.334}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.806, "count": 120, "max_ms": 56.065, "p95_ms": 41.775}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.766, "count": 120, "max_ms": 57.175, "p95_ms": 49.515}, "yolo11_640_fire_11class": {"avg_ms": 49.406, "count": 120, "max_ms": 56.048, "p95_ms": 51.843}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.867, "count": 120, "max_ms": 57.186, "p95_ms": 49.984}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.169, "count": 120, "max_ms": 85.029, "p95_ms": 83.396}, "read_interval": {"avg_ms": 316.531, "count": 120, "max_ms": 27002.62, "p95_ms": 96.256}, "triton_wait": {"avg_ms": 49.47, "count": 120, "max_ms": 57.196, "p95_ms": 51.855}, "visualize": {"avg_ms": 3.537, "count": 120, "max_ms": 4.181, "p95_ms": 3.962}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 15.136, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10469}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10477.668, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ee5acb0e00] reference picture missing during reorder [h264 @ 0x55ee5acb0e00] Missing reference picture, default is 65544 [h264 @ 0x55ee5ad7fb00] mmco: unref short failure [h264 @ 0x55ee5ad7fb00] illegal short term buffer state detected [h264 @ 0x55ee5a432980] error while decoding MB 35 9, bytestream -6 [h264 @ 0x55ee5a446b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee5a446b40] error while decoding MB 0 33, bytestream 2191 [h264 @ 0x55ee5a41e880] top block un", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 365, "h264_decode_error_count": 1195, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:52:50+09:00", "last_error_summary": "[h264 @ 0x55ee5acb0e00] reference picture missing during reorder [h264 @ 0x55ee5acb0e00] Missing reference picture, default is 65544 [h264 @ 0x55ee5ad7fb00] mmco: unref short failure [h264 @ 0x55ee5ad7fb00] illegal short term buffer state detected [h264 @ 0x55ee5a432980] error while decoding MB 35 9, bytestream -6 [h264 @ 0x55ee5a446b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee5a446b40] error while decoding MB 0 33, bytestream 2191 [h264 @ 0x55ee5a41e880] top block un", "last_exit_at": "2026-07-01T02:41:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 667.559, "last_started_at": "2026-07-01T02:41:42+09:00", "pid": 74211, "process": {"exists": false, "pid": 74211}, "restart_count": 398, "restart_count_per_hour": 4, "rtsp_timeout_count": 348, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 399}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.331, "sample_count": 2591, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1256374, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:52:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T02:52:50+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-01T02:52:50+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T02:52:50.223776+09:00", "mtime_epoch": 1782841970.223776, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 135482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:52:50+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4963.805, "name": "celery", "pid": 267, "rss_mb": 4963.805, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.27, "pushed": 3.27, "read": 3.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.765, "count": 98, "max_ms": 26.142, "p95_ms": 25.385}, "gstreamer_push": {"avg_ms": 1.221, "count": 98, "max_ms": 1.756, "p95_ms": 1.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.376, "count": 98, "max_ms": 43.248, "p95_ms": 41.789}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.162, "count": 98, "max_ms": 72.237, "p95_ms": 65.546}, "yolo11_640_fire_11class": {"avg_ms": 58.009, "count": 98, "max_ms": 76.462, "p95_ms": 69.199}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.566, "count": 98, "max_ms": 77.326, "p95_ms": 74.193}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.618, "count": 98, "max_ms": 105.913, "p95_ms": 103.281}, "read_interval": {"avg_ms": 306.168, "count": 98, "max_ms": 19614.332, "p95_ms": 119.939}, "triton_wait": {"avg_ms": 62.855, "count": 98, "max_ms": 77.338, "p95_ms": 74.205}, "visualize": {"avg_ms": 3.332, "count": 98, "max_ms": 4.48, "p95_ms": 3.644}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10469}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10477.207, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:52:50+09:00", "last_error_summary": "[h264 @ 0x55ee5acb0e00] reference picture missing during reorder [h264 @ 0x55ee5acb0e00] Missing reference picture, default is 65544 [h264 @ 0x55ee5ad7fb00] mmco: unref short failure [h264 @ 0x55ee5ad7fb00] illegal short term buffer state detected [h264 @ 0x55ee5a432980] error while decoding MB 35 9, bytestream -6 [h264 @ 0x55ee5a446b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee5a446b40] error while decoding MB 0 33, bytestream 2191 [h264 @ 0x55ee5a41e880] top block un", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 365, "h264_decode_error_count": 1195, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:52:50+09:00", "last_error_summary": "[h264 @ 0x55ee5acb0e00] reference picture missing during reorder [h264 @ 0x55ee5acb0e00] Missing reference picture, default is 65544 [h264 @ 0x55ee5ad7fb00] mmco: unref short failure [h264 @ 0x55ee5ad7fb00] illegal short term buffer state detected [h264 @ 0x55ee5a432980] error while decoding MB 35 9, bytestream -6 [h264 @ 0x55ee5a446b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee5a446b40] error while decoding MB 0 33, bytestream 2191 [h264 @ 0x55ee5a41e880] top block un", "last_exit_at": "2026-07-01T02:52:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 667.581, "last_started_at": "2026-07-01T02:41:42+09:00", "pid": 74211, "process": {"exists": false, "pid": 74211}, "restart_count": 398, "restart_count_per_hour": 4, "rtsp_timeout_count": 348, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 399}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.331, "sample_count": 2592, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1256375, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:52:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T02:52:50+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-01T02:52:50+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-01T02:52:50.223776+09:00", "mtime_epoch": 1782841970.223776, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 135482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:52:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4963.805, "name": "celery", "pid": 267, "rss_mb": 4963.805, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.824, "count": 99, "max_ms": 27.652, "p95_ms": 25.435}, "gstreamer_push": {"avg_ms": 1.221, "count": 99, "max_ms": 1.756, "p95_ms": 1.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.341, "count": 99, "max_ms": 43.248, "p95_ms": 41.789}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.047, "count": 99, "max_ms": 72.237, "p95_ms": 65.546}, "yolo11_640_fire_11class": {"avg_ms": 57.872, "count": 99, "max_ms": 76.462, "p95_ms": 69.199}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.402, "count": 99, "max_ms": 77.326, "p95_ms": 74.193}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.518, "count": 99, "max_ms": 105.913, "p95_ms": 103.281}, "read_interval": {"avg_ms": 306.168, "count": 98, "max_ms": 19614.332, "p95_ms": 119.939}, "triton_wait": {"avg_ms": 62.689, "count": 99, "max_ms": 77.338, "p95_ms": 74.205}, "visualize": {"avg_ms": 3.338, "count": 99, "max_ms": 4.48, "p95_ms": 3.666}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10469}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10477.152, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 365, "h264_decode_error_count": 1195, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:52:50+09:00", "last_error_summary": "[h264 @ 0x55ee5acb0e00] reference picture missing during reorder [h264 @ 0x55ee5acb0e00] Missing reference picture, default is 65544 [h264 @ 0x55ee5ad7fb00] mmco: unref short failure [h264 @ 0x55ee5ad7fb00] illegal short term buffer state detected [h264 @ 0x55ee5a432980] error while decoding MB 35 9, bytestream -6 [h264 @ 0x55ee5a446b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee5a446b40] error while decoding MB 0 33, bytestream 2191 [h264 @ 0x55ee5a41e880] top block un", "last_exit_at": "2026-07-01T02:52:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 667.589, "last_started_at": "2026-07-01T02:41:42+09:00", "pid": 74211, "process": {"exists": false, "pid": 74211}, "restart_count": 398, "restart_count_per_hour": 4, "rtsp_timeout_count": 348, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 399}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.331, "sample_count": 2592, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1256375, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:52:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T02:52:50+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-01T02:52:50+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T02:52:50.223776+09:00", "mtime_epoch": 1782841970.223776, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 135482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:52:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4963.805, "name": "celery", "pid": 267, "rss_mb": 4963.805, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.267, "pushed": 3.3, "read": 3.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.83, "count": 98, "max_ms": 27.652, "p95_ms": 25.435}, "gstreamer_push": {"avg_ms": 1.221, "count": 99, "max_ms": 1.756, "p95_ms": 1.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.296, "count": 98, "max_ms": 43.248, "p95_ms": 41.789}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.985, "count": 98, "max_ms": 72.237, "p95_ms": 65.546}, "yolo11_640_fire_11class": {"avg_ms": 57.796, "count": 98, "max_ms": 76.462, "p95_ms": 69.199}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.307, "count": 98, "max_ms": 77.326, "p95_ms": 74.193}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.428, "count": 98, "max_ms": 105.913, "p95_ms": 103.281}, "read_interval": {"avg_ms": 306.168, "count": 98, "max_ms": 19614.332, "p95_ms": 119.939}, "triton_wait": {"avg_ms": 62.597, "count": 98, "max_ms": 77.338, "p95_ms": 74.205}, "visualize": {"avg_ms": 3.338, "count": 98, "max_ms": 4.48, "p95_ms": 3.666}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10469}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10595.5, "limit_mb": 8796093022207.996}, "detail": {"pid": 74622, "restart_count": 399, "start_count": 400}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 365, "h264_decode_error_count": 1195, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:52:50+09:00", "last_error_summary": "[h264 @ 0x55ee5acb0e00] reference picture missing during reorder [h264 @ 0x55ee5acb0e00] Missing reference picture, default is 65544 [h264 @ 0x55ee5ad7fb00] mmco: unref short failure [h264 @ 0x55ee5ad7fb00] illegal short term buffer state detected [h264 @ 0x55ee5a432980] error while decoding MB 35 9, bytestream -6 [h264 @ 0x55ee5a446b40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee5a446b40] error while decoding MB 0 33, bytestream 2191 [h264 @ 0x55ee5a41e880] top block un", "last_exit_at": "2026-07-01T02:52:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:52:53+09:00", "pid": 74622, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.07, "name": "ffmpeg", "pid": 74622, "rss_mb": 149.07, "state": "S (sleeping)", "threads": 106}, "restart_count": 399, "restart_count_per_hour": 5, "rtsp_timeout_count": 348, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 400}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.339, "sample_count": 2550, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1256375, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:52:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.922, "last_frame_pushed_at": "2026-07-01T02:52:50+09:00", "last_frame_read_age_sec": 5.007, "last_frame_read_at": "2026-07-01T02:52:50+09:00", "mjpeg": {"age_sec": 4.903, "exists": true, "mtime": "2026-07-01T02:52:50.319778+09:00", "mtime_epoch": 1782841970.3197777, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 133996}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:52:55+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4963.805, "name": "celery", "pid": 267, "rss_mb": 4963.805, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.871, "pushed": 1.871, "read": 1.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.447, "count": 56, "max_ms": 27.652, "p95_ms": 25.991}, "gstreamer_push": {"avg_ms": 1.234, "count": 56, "max_ms": 1.756, "p95_ms": 1.475}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.368, "count": 56, "max_ms": 43.248, "p95_ms": 41.492}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.298, "count": 56, "max_ms": 64.456, "p95_ms": 63.425}, "yolo11_640_fire_11class": {"avg_ms": 51.955, "count": 56, "max_ms": 69.164, "p95_ms": 65.501}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.288, "count": 56, "max_ms": 73.615, "p95_ms": 72.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.124, "count": 56, "max_ms": 102.69, "p95_ms": 101.823}, "read_interval": {"avg_ms": 455.011, "count": 55, "max_ms": 19614.332, "p95_ms": 117.877}, "triton_wait": {"avg_ms": 55.743, "count": 56, "max_ms": 73.628, "p95_ms": 72.49}, "visualize": {"avg_ms": 3.303, "count": 56, "max_ms": 4.48, "p95_ms": 3.908}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10479.566, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557998e2b140] error while decoding MB 77 5, bytestream -16 [h264 @ 0x557999af0700] left block unavailable for requested intra mode [h264 @ 0x557999af0700] error while decoding MB 0 8, bytestream 12459 [h264 @ 0x557998e2b140] error while decoding MB 13 20, bytestream -5 [h264 @ 0x557998f38f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557998f38f40] error while decoding MB 0 20, bytestream 6644 [in#0/rtsp @ 0x557998dfbe00] Error during demuxing: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 366, "h264_decode_error_count": 1199, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:57:52+09:00", "last_error_summary": "[h264 @ 0x557998e2b140] error while decoding MB 77 5, bytestream -16 [h264 @ 0x557999af0700] left block unavailable for requested intra mode [h264 @ 0x557999af0700] error while decoding MB 0 8, bytestream 12459 [h264 @ 0x557998e2b140] error while decoding MB 13 20, bytestream -5 [h264 @ 0x557998f38f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557998f38f40] error while decoding MB 0 20, bytestream 6644 [in#0/rtsp @ 0x557998dfbe00] Error during demuxing: Connection timed out", "last_exit_at": "2026-07-01T02:52:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 299.07, "last_started_at": "2026-07-01T02:52:53+09:00", "pid": 74622, "process": {"exists": false, "pid": 74622}, "restart_count": 399, "restart_count_per_hour": 4, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 400}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.926, "sample_count": 2437, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1258812, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:57:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.084, "last_frame_pushed_at": "2026-07-01T02:57:52+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-01T02:57:52+09:00", "mjpeg": {"age_sec": 0.067, "exists": true, "mtime": "2026-07-01T02:57:52.225160+09:00", "mtime_epoch": 1782842272.22516, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117130}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:57:52+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4965.094, "name": "celery", "pid": 267, "rss_mb": 4965.094, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.211, "pushed": 3.211, "read": 3.211}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.285, "count": 96, "max_ms": 24.769, "p95_ms": 24.11}, "gstreamer_push": {"avg_ms": 1.193, "count": 96, "max_ms": 1.305, "p95_ms": 1.263}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.427, "count": 96, "max_ms": 44.347, "p95_ms": 41.923}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.916, "count": 96, "max_ms": 69.365, "p95_ms": 67.917}, "yolo11_640_fire_11class": {"avg_ms": 62.152, "count": 96, "max_ms": 75.84, "p95_ms": 68.322}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.883, "count": 96, "max_ms": 78.194, "p95_ms": 77.321}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.185, "count": 96, "max_ms": 107.678, "p95_ms": 106.265}, "read_interval": {"avg_ms": 311.768, "count": 96, "max_ms": 10126.355, "p95_ms": 121.307}, "triton_wait": {"avg_ms": 69.902, "count": 96, "max_ms": 78.206, "p95_ms": 77.333}, "visualize": {"avg_ms": 3.342, "count": 96, "max_ms": 3.728, "p95_ms": 3.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10479.547, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T02:57:52+09:00", "last_error_summary": "[h264 @ 0x557998e2b140] error while decoding MB 77 5, bytestream -16 [h264 @ 0x557999af0700] left block unavailable for requested intra mode [h264 @ 0x557999af0700] error while decoding MB 0 8, bytestream 12459 [h264 @ 0x557998e2b140] error while decoding MB 13 20, bytestream -5 [h264 @ 0x557998f38f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557998f38f40] error while decoding MB 0 20, bytestream 6644 [in#0/rtsp @ 0x557998dfbe00] Error during demuxing: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 366, "h264_decode_error_count": 1199, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:57:52+09:00", "last_error_summary": "[h264 @ 0x557998e2b140] error while decoding MB 77 5, bytestream -16 [h264 @ 0x557999af0700] left block unavailable for requested intra mode [h264 @ 0x557999af0700] error while decoding MB 0 8, bytestream 12459 [h264 @ 0x557998e2b140] error while decoding MB 13 20, bytestream -5 [h264 @ 0x557998f38f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557998f38f40] error while decoding MB 0 20, bytestream 6644 [in#0/rtsp @ 0x557998dfbe00] Error during demuxing: Connection timed out", "last_exit_at": "2026-07-01T02:57:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 299.095, "last_started_at": "2026-07-01T02:52:53+09:00", "pid": 74622, "process": {"exists": false, "pid": 74622}, "restart_count": 399, "restart_count_per_hour": 4, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 400}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.926, "sample_count": 2438, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1258813, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:57:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T02:57:52+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-01T02:57:52+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-01T02:57:52.225160+09:00", "mtime_epoch": 1782842272.22516, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117130}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:57:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4965.094, "name": "celery", "pid": 267, "rss_mb": 4965.094, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.242, "pushed": 3.242, "read": 3.208}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.339, "count": 97, "max_ms": 26.581, "p95_ms": 24.265}, "gstreamer_push": {"avg_ms": 1.197, "count": 97, "max_ms": 1.5, "p95_ms": 1.268}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.392, "count": 97, "max_ms": 44.347, "p95_ms": 41.923}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.796, "count": 97, "max_ms": 69.365, "p95_ms": 67.917}, "yolo11_640_fire_11class": {"avg_ms": 62.024, "count": 97, "max_ms": 75.84, "p95_ms": 68.322}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.688, "count": 97, "max_ms": 78.194, "p95_ms": 77.321}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.064, "count": 97, "max_ms": 107.678, "p95_ms": 106.265}, "read_interval": {"avg_ms": 311.768, "count": 96, "max_ms": 10126.355, "p95_ms": 121.307}, "triton_wait": {"avg_ms": 69.707, "count": 97, "max_ms": 78.206, "p95_ms": 77.333}, "visualize": {"avg_ms": 3.355, "count": 97, "max_ms": 4.555, "p95_ms": 3.533}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10479.535, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 366, "h264_decode_error_count": 1199, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:57:52+09:00", "last_error_summary": "[h264 @ 0x557998e2b140] error while decoding MB 77 5, bytestream -16 [h264 @ 0x557999af0700] left block unavailable for requested intra mode [h264 @ 0x557999af0700] error while decoding MB 0 8, bytestream 12459 [h264 @ 0x557998e2b140] error while decoding MB 13 20, bytestream -5 [h264 @ 0x557998f38f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557998f38f40] error while decoding MB 0 20, bytestream 6644 [in#0/rtsp @ 0x557998dfbe00] Error during demuxing: Connection timed out", "last_exit_at": "2026-07-01T02:57:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 299.103, "last_started_at": "2026-07-01T02:52:53+09:00", "pid": 74622, "process": {"exists": false, "pid": 74622}, "restart_count": 399, "restart_count_per_hour": 4, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 400}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.926, "sample_count": 2438, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1258813, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:57:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T02:57:52+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T02:57:52+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-01T02:57:52.225160+09:00", "mtime_epoch": 1782842272.22516, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117130}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:57:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4965.094, "name": "celery", "pid": 267, "rss_mb": 4965.094, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.241, "pushed": 3.241, "read": 3.207}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.339, "count": 97, "max_ms": 26.581, "p95_ms": 24.265}, "gstreamer_push": {"avg_ms": 1.197, "count": 97, "max_ms": 1.5, "p95_ms": 1.268}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.392, "count": 97, "max_ms": 44.347, "p95_ms": 41.923}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.796, "count": 97, "max_ms": 69.365, "p95_ms": 67.917}, "yolo11_640_fire_11class": {"avg_ms": 62.024, "count": 97, "max_ms": 75.84, "p95_ms": 68.322}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.688, "count": 97, "max_ms": 78.194, "p95_ms": 77.321}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.064, "count": 97, "max_ms": 107.678, "p95_ms": 106.265}, "read_interval": {"avg_ms": 311.768, "count": 96, "max_ms": 10126.355, "p95_ms": 121.307}, "triton_wait": {"avg_ms": 69.707, "count": 97, "max_ms": 78.206, "p95_ms": 77.333}, "visualize": {"avg_ms": 3.355, "count": 97, "max_ms": 4.555, "p95_ms": 3.533}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10591.109, "limit_mb": 8796093022207.996}, "detail": {"pid": 74729, "restart_count": 400, "start_count": 401}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 366, "h264_decode_error_count": 1199, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T02:57:52+09:00", "last_error_summary": "[h264 @ 0x557998e2b140] error while decoding MB 77 5, bytestream -16 [h264 @ 0x557999af0700] left block unavailable for requested intra mode [h264 @ 0x557999af0700] error while decoding MB 0 8, bytestream 12459 [h264 @ 0x557998e2b140] error while decoding MB 13 20, bytestream -5 [h264 @ 0x557998f38f40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557998f38f40] error while decoding MB 0 20, bytestream 6644 [in#0/rtsp @ 0x557998dfbe00] Error during demuxing: Connection timed out", "last_exit_at": "2026-07-01T02:57:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T02:57:54+09:00", "pid": 74729, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.723, "name": "ffmpeg", "pid": 74729, "rss_mb": 141.723, "state": "S (sleeping)", "threads": 106}, "restart_count": 400, "restart_count_per_hour": 5, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 401}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.931, "sample_count": 2419, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1258813, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T02:57:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.656, "last_frame_pushed_at": "2026-07-01T02:57:52+09:00", "last_frame_read_age_sec": 4.745, "last_frame_read_at": "2026-07-01T02:57:52+09:00", "mjpeg": {"age_sec": 4.641, "exists": true, "mtime": "2026-07-01T02:57:52.325162+09:00", "mtime_epoch": 1782842272.3251617, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116752}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T02:57:56+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4965.094, "name": "celery", "pid": 267, "rss_mb": 4965.094, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 56, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.904, "pushed": 1.904, "read": 1.87}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.622, "count": 57, "max_ms": 26.581, "p95_ms": 24.72}, "gstreamer_push": {"avg_ms": 1.207, "count": 57, "max_ms": 1.5, "p95_ms": 1.273}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.251, "count": 57, "max_ms": 44.347, "p95_ms": 42.435}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.406, "count": 57, "max_ms": 69.365, "p95_ms": 67.33}, "yolo11_640_fire_11class": {"avg_ms": 60.604, "count": 57, "max_ms": 70.151, "p95_ms": 68.174}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.884, "count": 57, "max_ms": 78.194, "p95_ms": 76.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.61, "count": 57, "max_ms": 107.678, "p95_ms": 105.518}, "read_interval": {"avg_ms": 451.804, "count": 56, "max_ms": 10126.355, "p95_ms": 122.337}, "triton_wait": {"avg_ms": 67.908, "count": 57, "max_ms": 78.206, "p95_ms": 76.229}, "visualize": {"avg_ms": 3.379, "count": 57, "max_ms": 4.555, "p95_ms": 3.615}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10503}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10482.289, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55940a5a5940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55940a5a5940] error while decoding MB 0 6, bytestream 3831 [h264 @ 0x55940a6747c0] reference picture missing during reorder [h264 @ 0x55940a6747c0] Missing reference picture, default is 65552 [h264 @ 0x55940a743640] mmco: unref short failure [h264 @ 0x55940a743640] illegal short term buffer state detected [h264 @ 0x559409c49940] error while decoding MB 61 29, bytestream -12 [h264 @ 0x55940a5a5940] cabac decod", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 367, "h264_decode_error_count": 1203, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:01:13+09:00", "last_error_summary": "[h264 @ 0x55940a5a5940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55940a5a5940] error while decoding MB 0 6, bytestream 3831 [h264 @ 0x55940a6747c0] reference picture missing during reorder [h264 @ 0x55940a6747c0] Missing reference picture, default is 65552 [h264 @ 0x55940a743640] mmco: unref short failure [h264 @ 0x55940a743640] illegal short term buffer state detected [h264 @ 0x559409c49940] error while decoding MB 61 29, bytestream -12 [h264 @ 0x55940a5a5940] cabac decod", "last_exit_at": "2026-07-01T02:57:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 198.289, "last_started_at": "2026-07-01T02:57:54+09:00", "pid": 74729, "process": {"exists": false, "pid": 74729}, "restart_count": 400, "restart_count_per_hour": 5, "rtsp_timeout_count": 352, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 401}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.806, "sample_count": 2258, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1260388, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:01:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-01T03:01:13+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-01T03:01:13+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-01T03:01:13.197001+09:00", "mtime_epoch": 1782842473.1970007, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118226}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:01:13+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4965.867, "name": "celery", "pid": 267, "rss_mb": 4965.867, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.502, "pushed": 3.502, "read": 3.502}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.224, "count": 105, "max_ms": 23.248, "p95_ms": 22.661}, "gstreamer_push": {"avg_ms": 1.219, "count": 105, "max_ms": 1.366, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.756, "count": 105, "max_ms": 42.665, "p95_ms": 41.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.841, "count": 105, "max_ms": 68.665, "p95_ms": 65.723}, "yolo11_640_fire_11class": {"avg_ms": 53.732, "count": 105, "max_ms": 72.416, "p95_ms": 66.534}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.19, "count": 105, "max_ms": 76.965, "p95_ms": 74.912}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.565, "count": 105, "max_ms": 106.224, "p95_ms": 104.214}, "read_interval": {"avg_ms": 285.964, "count": 105, "max_ms": 10131.579, "p95_ms": 120.07}, "triton_wait": {"avg_ms": 58.243, "count": 105, "max_ms": 76.977, "p95_ms": 74.925}, "visualize": {"avg_ms": 3.406, "count": 105, "max_ms": 3.928, "p95_ms": 3.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10503}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10482.211, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:01:13+09:00", "last_error_summary": "[h264 @ 0x55940a5a5940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55940a5a5940] error while decoding MB 0 6, bytestream 3831 [h264 @ 0x55940a6747c0] reference picture missing during reorder [h264 @ 0x55940a6747c0] Missing reference picture, default is 65552 [h264 @ 0x55940a743640] mmco: unref short failure [h264 @ 0x55940a743640] illegal short term buffer state detected [h264 @ 0x559409c49940] error while decoding MB 61 29, bytestream -12 [h264 @ 0x55940a5a5940] cabac decod", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 367, "h264_decode_error_count": 1203, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:01:13+09:00", "last_error_summary": "[h264 @ 0x55940a5a5940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55940a5a5940] error while decoding MB 0 6, bytestream 3831 [h264 @ 0x55940a6747c0] reference picture missing during reorder [h264 @ 0x55940a6747c0] Missing reference picture, default is 65552 [h264 @ 0x55940a743640] mmco: unref short failure [h264 @ 0x55940a743640] illegal short term buffer state detected [h264 @ 0x559409c49940] error while decoding MB 61 29, bytestream -12 [h264 @ 0x55940a5a5940] cabac decod", "last_exit_at": "2026-07-01T03:01:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 198.312, "last_started_at": "2026-07-01T02:57:54+09:00", "pid": 74729, "process": {"exists": false, "pid": 74729}, "restart_count": 400, "restart_count_per_hour": 5, "rtsp_timeout_count": 352, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 401}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.806, "sample_count": 2259, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1260389, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:01:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T03:01:13+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-01T03:01:13+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-01T03:01:13.197001+09:00", "mtime_epoch": 1782842473.1970007, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118226}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:01:13+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4965.867, "name": "celery", "pid": 267, "rss_mb": 4965.867, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.501, "pushed": 3.501, "read": 3.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.237, "count": 105, "max_ms": 23.248, "p95_ms": 22.661}, "gstreamer_push": {"avg_ms": 1.22, "count": 105, "max_ms": 1.366, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.757, "count": 105, "max_ms": 42.665, "p95_ms": 41.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.61, "count": 105, "max_ms": 68.665, "p95_ms": 65.334}, "yolo11_640_fire_11class": {"avg_ms": 53.512, "count": 105, "max_ms": 72.416, "p95_ms": 66.483}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.886, "count": 105, "max_ms": 76.499, "p95_ms": 74.898}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.291, "count": 105, "max_ms": 106.224, "p95_ms": 104.2}, "read_interval": {"avg_ms": 285.964, "count": 105, "max_ms": 10131.579, "p95_ms": 120.07}, "triton_wait": {"avg_ms": 57.939, "count": 105, "max_ms": 76.512, "p95_ms": 74.91}, "visualize": {"avg_ms": 3.418, "count": 105, "max_ms": 4.507, "p95_ms": 3.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10503}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10482.039, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 367, "h264_decode_error_count": 1203, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:01:13+09:00", "last_error_summary": "[h264 @ 0x55940a5a5940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55940a5a5940] error while decoding MB 0 6, bytestream 3831 [h264 @ 0x55940a6747c0] reference picture missing during reorder [h264 @ 0x55940a6747c0] Missing reference picture, default is 65552 [h264 @ 0x55940a743640] mmco: unref short failure [h264 @ 0x55940a743640] illegal short term buffer state detected [h264 @ 0x559409c49940] error while decoding MB 61 29, bytestream -12 [h264 @ 0x55940a5a5940] cabac decod", "last_exit_at": "2026-07-01T03:01:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 198.319, "last_started_at": "2026-07-01T02:57:54+09:00", "pid": 74729, "process": {"exists": false, "pid": 74729}, "restart_count": 400, "restart_count_per_hour": 5, "rtsp_timeout_count": 352, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 401}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.806, "sample_count": 2259, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1260389, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:01:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T03:01:13+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-01T03:01:13+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T03:01:13.197001+09:00", "mtime_epoch": 1782842473.1970007, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118226}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:01:13+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4965.867, "name": "celery", "pid": 267, "rss_mb": 4965.867, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.5, "pushed": 3.5, "read": 3.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.237, "count": 105, "max_ms": 23.248, "p95_ms": 22.661}, "gstreamer_push": {"avg_ms": 1.22, "count": 105, "max_ms": 1.366, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.757, "count": 105, "max_ms": 42.665, "p95_ms": 41.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.61, "count": 105, "max_ms": 68.665, "p95_ms": 65.334}, "yolo11_640_fire_11class": {"avg_ms": 53.512, "count": 105, "max_ms": 72.416, "p95_ms": 66.483}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.886, "count": 105, "max_ms": 76.499, "p95_ms": 74.898}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.291, "count": 105, "max_ms": 106.224, "p95_ms": 104.2}, "read_interval": {"avg_ms": 285.964, "count": 105, "max_ms": 10131.579, "p95_ms": 120.07}, "triton_wait": {"avg_ms": 57.939, "count": 105, "max_ms": 76.512, "p95_ms": 74.91}, "visualize": {"avg_ms": 3.418, "count": 105, "max_ms": 4.507, "p95_ms": 3.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10503}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10590.055, "limit_mb": 8796093022207.996}, "detail": {"pid": 74836, "restart_count": 401, "start_count": 402}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 367, "h264_decode_error_count": 1203, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:01:13+09:00", "last_error_summary": "[h264 @ 0x55940a5a5940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55940a5a5940] error while decoding MB 0 6, bytestream 3831 [h264 @ 0x55940a6747c0] reference picture missing during reorder [h264 @ 0x55940a6747c0] Missing reference picture, default is 65552 [h264 @ 0x55940a743640] mmco: unref short failure [h264 @ 0x55940a743640] illegal short term buffer state detected [h264 @ 0x559409c49940] error while decoding MB 61 29, bytestream -12 [h264 @ 0x55940a5a5940] cabac decod", "last_exit_at": "2026-07-01T03:01:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:01:16+09:00", "pid": 74836, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.449, "name": "ffmpeg", "pid": 74836, "rss_mb": 138.449, "state": "S (sleeping)", "threads": 106}, "restart_count": 401, "restart_count_per_hour": 6, "rtsp_timeout_count": 352, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 402}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.781, "sample_count": 2219, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1260389, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:01:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.806, "last_frame_pushed_at": "2026-07-01T03:01:13+09:00", "last_frame_read_age_sec": 4.886, "last_frame_read_at": "2026-07-01T03:01:13+09:00", "mjpeg": {"age_sec": 4.793, "exists": true, "mtime": "2026-07-01T03:01:13.285002+09:00", "mtime_epoch": 1782842473.2850025, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118374}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:01:18+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4965.867, "name": "celery", "pid": 267, "rss_mb": 4965.867, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.138, "pushed": 2.138, "read": 2.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.099, "count": 64, "max_ms": 23.248, "p95_ms": 22.624}, "gstreamer_push": {"avg_ms": 1.226, "count": 64, "max_ms": 1.35, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.458, "count": 64, "max_ms": 41.717, "p95_ms": 40.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.512, "count": 64, "max_ms": 68.665, "p95_ms": 63.446}, "yolo11_640_fire_11class": {"avg_ms": 47.211, "count": 64, "max_ms": 72.416, "p95_ms": 65.446}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.206, "count": 64, "max_ms": 75.104, "p95_ms": 72.397}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.449, "count": 64, "max_ms": 104.303, "p95_ms": 101.555}, "read_interval": {"avg_ms": 399.215, "count": 63, "max_ms": 10131.579, "p95_ms": 117.604}, "triton_wait": {"avg_ms": 49.287, "count": 64, "max_ms": 75.116, "p95_ms": 72.41}, "visualize": {"avg_ms": 3.439, "count": 64, "max_ms": 4.507, "p95_ms": 3.836}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10309.852, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5616f2e9f080] error while decoding MB 118 19, bytestream -7 [in#0/rtsp @ 0x5616f2849e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5616f2849e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 171, "h264_decode_error_count": 176, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:01:19+09:00", "last_error_summary": "[h264 @ 0x5616f2e9f080] error while decoding MB 118 19, bytestream -7 [in#0/rtsp @ 0x5616f2849e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5616f2849e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T02:44:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 749.065, "last_started_at": "2026-07-01T02:48:50+09:00", "pid": 74470, "process": {"exists": false, "pid": 74470}, "restart_count": 170, "restart_count_per_hour": 10, "rtsp_timeout_count": 278, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 171}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.921, "sample_count": 2431, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1399057, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:01:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-01T03:01:19+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-01T03:01:19+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T03:01:19.553122+09:00", "mtime_epoch": 1782842479.5531218, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86133}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:01:19+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1720.816, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.68, "pushed": 0.68, "read": 0.737}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.991, "count": 12, "max_ms": 23.198, "p95_ms": 23.198}, "gstreamer_push": {"avg_ms": 1.232, "count": 12, "max_ms": 1.346, "p95_ms": 1.346}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.315, "count": 12, "max_ms": 44.097, "p95_ms": 44.097}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.465, "count": 12, "max_ms": 54.221, "p95_ms": 54.221}, "yolo11_640_fire_11class": {"avg_ms": 49.374, "count": 12, "max_ms": 57.346, "p95_ms": 57.346}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.267, "count": 12, "max_ms": 53.169, "p95_ms": 53.169}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.343, "count": 12, "max_ms": 90.321, "p95_ms": 90.321}, "read_interval": {"avg_ms": 2344.073, "count": 13, "max_ms": 16575.202, "p95_ms": 16575.202}, "triton_wait": {"avg_ms": 49.984, "count": 12, "max_ms": 57.357, "p95_ms": 57.357}, "visualize": {"avg_ms": 3.517, "count": 12, "max_ms": 4.401, "p95_ms": 4.401}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.641, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10312.125, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:01:19+09:00", "last_error_summary": "[h264 @ 0x5616f2e9f080] error while decoding MB 118 19, bytestream -7 [in#0/rtsp @ 0x5616f2849e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5616f2849e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 171, "h264_decode_error_count": 176, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:01:19+09:00", "last_error_summary": "[h264 @ 0x5616f2e9f080] error while decoding MB 118 19, bytestream -7 [in#0/rtsp @ 0x5616f2849e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5616f2849e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:01:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 749.09, "last_started_at": "2026-07-01T02:48:50+09:00", "pid": 74470, "process": {"exists": false, "pid": 74470}, "restart_count": 170, "restart_count_per_hour": 10, "rtsp_timeout_count": 278, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 171}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.921, "sample_count": 2432, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1399058, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:01:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T03:01:19+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-01T03:01:19+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T03:01:19.553122+09:00", "mtime_epoch": 1782842479.5531218, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86133}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:01:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1720.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.736, "pushed": 0.736, "read": 0.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.244, "count": 13, "max_ms": 23.198, "p95_ms": 23.198}, "gstreamer_push": {"avg_ms": 1.243, "count": 13, "max_ms": 1.375, "p95_ms": 1.375}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.293, "count": 13, "max_ms": 44.097, "p95_ms": 44.097}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.468, "count": 13, "max_ms": 54.221, "p95_ms": 54.221}, "yolo11_640_fire_11class": {"avg_ms": 49.395, "count": 13, "max_ms": 57.346, "p95_ms": 57.346}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.165, "count": 13, "max_ms": 53.169, "p95_ms": 53.169}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.874, "count": 13, "max_ms": 90.321, "p95_ms": 90.321}, "read_interval": {"avg_ms": 2344.073, "count": 13, "max_ms": 16575.202, "p95_ms": 16575.202}, "triton_wait": {"avg_ms": 49.959, "count": 13, "max_ms": 57.357, "p95_ms": 57.357}, "visualize": {"avg_ms": 3.78, "count": 13, "max_ms": 6.945, "p95_ms": 6.945}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.665, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10312.094, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 171, "h264_decode_error_count": 176, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:01:19+09:00", "last_error_summary": "[h264 @ 0x5616f2e9f080] error while decoding MB 118 19, bytestream -7 [in#0/rtsp @ 0x5616f2849e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5616f2849e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:01:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 749.097, "last_started_at": "2026-07-01T02:48:50+09:00", "pid": 74470, "process": {"exists": false, "pid": 74470}, "restart_count": 170, "restart_count_per_hour": 10, "rtsp_timeout_count": 278, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 171}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.921, "sample_count": 2432, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1399058, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:01:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T03:01:19+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-01T03:01:19+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-01T03:01:19.553122+09:00", "mtime_epoch": 1782842479.5531218, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86133}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:01:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1720.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.736, "pushed": 0.736, "read": 0.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.244, "count": 13, "max_ms": 23.198, "p95_ms": 23.198}, "gstreamer_push": {"avg_ms": 1.243, "count": 13, "max_ms": 1.375, "p95_ms": 1.375}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.293, "count": 13, "max_ms": 44.097, "p95_ms": 44.097}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.468, "count": 13, "max_ms": 54.221, "p95_ms": 54.221}, "yolo11_640_fire_11class": {"avg_ms": 49.395, "count": 13, "max_ms": 57.346, "p95_ms": 57.346}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.165, "count": 13, "max_ms": 53.169, "p95_ms": 53.169}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.874, "count": 13, "max_ms": 90.321, "p95_ms": 90.321}, "read_interval": {"avg_ms": 2344.073, "count": 13, "max_ms": 16575.202, "p95_ms": 16575.202}, "triton_wait": {"avg_ms": 49.959, "count": 13, "max_ms": 57.357, "p95_ms": 57.357}, "visualize": {"avg_ms": 3.78, "count": 13, "max_ms": 6.945, "p95_ms": 6.945}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.673, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10593.891, "limit_mb": 8796093022207.996}, "detail": {"pid": 74943, "restart_count": 171, "start_count": 172}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 171, "h264_decode_error_count": 176, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:01:19+09:00", "last_error_summary": "[h264 @ 0x5616f2e9f080] error while decoding MB 118 19, bytestream -7 [in#0/rtsp @ 0x5616f2849e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5616f2849e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:01:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:01:22+09:00", "pid": 74943, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.09, "name": "ffmpeg", "pid": 74943, "rss_mb": 204.09, "state": "S (sleeping)", "threads": 106}, "restart_count": 171, "restart_count_per_hour": 11, "rtsp_timeout_count": 278, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 172}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.971, "sample_count": 2393, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1399058, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:01:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.71, "last_frame_pushed_at": "2026-07-01T03:01:19+09:00", "last_frame_read_age_sec": 4.797, "last_frame_read_at": "2026-07-01T03:01:19+09:00", "mjpeg": {"age_sec": 4.696, "exists": true, "mtime": "2026-07-01T03:01:19.649124+09:00", "mtime_epoch": 1782842479.6491237, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86646}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:01:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1720.816, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.581, "pushed": 0.581, "read": 0.581}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.244, "count": 13, "max_ms": 23.198, "p95_ms": 23.198}, "gstreamer_push": {"avg_ms": 1.243, "count": 13, "max_ms": 1.375, "p95_ms": 1.375}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.293, "count": 13, "max_ms": 44.097, "p95_ms": 44.097}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.468, "count": 13, "max_ms": 54.221, "p95_ms": 54.221}, "yolo11_640_fire_11class": {"avg_ms": 49.395, "count": 13, "max_ms": 57.346, "p95_ms": 57.346}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.165, "count": 13, "max_ms": 53.169, "p95_ms": 53.169}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.874, "count": 13, "max_ms": 90.321, "p95_ms": 90.321}, "read_interval": {"avg_ms": 2344.073, "count": 13, "max_ms": 16575.202, "p95_ms": 16575.202}, "triton_wait": {"avg_ms": 49.959, "count": 13, "max_ms": 57.357, "p95_ms": 57.357}, "visualize": {"avg_ms": 3.78, "count": 13, "max_ms": 6.945, "p95_ms": 6.945}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.37, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11687}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10444.32, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56112f7a1380] error while decoding MB 77 32, bytestream -21 [in#0/rtsp @ 0x56112f07be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56112f07be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 172, "h264_decode_error_count": 177, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:07:53+09:00", "last_error_summary": "[h264 @ 0x56112f7a1380] error while decoding MB 77 32, bytestream -21 [in#0/rtsp @ 0x56112f07be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56112f07be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:01:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 390.962, "last_started_at": "2026-07-01T03:01:22+09:00", "pid": 74943, "process": {"exists": false, "pid": 74943}, "restart_count": 171, "restart_count_per_hour": 10, "rtsp_timeout_count": 280, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 172}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.601, "sample_count": 2587, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1402505, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:07:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-01T03:07:53+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-01T03:07:53+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-01T03:07:53.248497+09:00", "mtime_epoch": 1782842873.248497, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90279}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:07:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1722.363, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 36, "pushed_frames": 36, "read_frames": 36, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.201, "pushed": 1.201, "read": 1.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.708, "count": 36, "max_ms": 20.231, "p95_ms": 20.118}, "gstreamer_push": {"avg_ms": 1.236, "count": 36, "max_ms": 1.326, "p95_ms": 1.311}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.308, "count": 36, "max_ms": 59.531, "p95_ms": 57.866}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.763, "count": 36, "max_ms": 67.917, "p95_ms": 66.914}, "yolo11_640_fire_11class": {"avg_ms": 66.937, "count": 36, "max_ms": 73.268, "p95_ms": 73.108}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.971, "count": 36, "max_ms": 76.907, "p95_ms": 76.438}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.968, "count": 36, "max_ms": 103.698, "p95_ms": 103.164}, "read_interval": {"avg_ms": 833.442, "count": 36, "max_ms": 16119.375, "p95_ms": 9969.823}, "triton_wait": {"avg_ms": 73.089, "count": 36, "max_ms": 76.919, "p95_ms": 76.45}, "visualize": {"avg_ms": 3.454, "count": 36, "max_ms": 3.604, "p95_ms": 3.569}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11687}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10443.906, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:07:53+09:00", "last_error_summary": "[h264 @ 0x56112f7a1380] error while decoding MB 77 32, bytestream -21 [in#0/rtsp @ 0x56112f07be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56112f07be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 172, "h264_decode_error_count": 177, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:07:53+09:00", "last_error_summary": "[h264 @ 0x56112f7a1380] error while decoding MB 77 32, bytestream -21 [in#0/rtsp @ 0x56112f07be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56112f07be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:07:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 391.005, "last_started_at": "2026-07-01T03:01:22+09:00", "pid": 74943, "process": {"exists": false, "pid": 74943}, "restart_count": 171, "restart_count_per_hour": 10, "rtsp_timeout_count": 280, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 172}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.601, "sample_count": 2588, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1402506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:07:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T03:07:53+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-01T03:07:53+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-01T03:07:53.248497+09:00", "mtime_epoch": 1782842873.248497, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90279}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:07:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1722.363, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 36, "pushed_frames": 36, "read_frames": 36, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.2, "pushed": 1.2, "read": 1.2}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.733, "count": 36, "max_ms": 20.231, "p95_ms": 20.118}, "gstreamer_push": {"avg_ms": 1.243, "count": 36, "max_ms": 1.403, "p95_ms": 1.326}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.106, "count": 36, "max_ms": 59.531, "p95_ms": 57.866}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.811, "count": 36, "max_ms": 67.917, "p95_ms": 67.311}, "yolo11_640_fire_11class": {"avg_ms": 66.979, "count": 36, "max_ms": 73.268, "p95_ms": 73.108}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.819, "count": 36, "max_ms": 76.907, "p95_ms": 76.438}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.876, "count": 36, "max_ms": 103.698, "p95_ms": 103.164}, "read_interval": {"avg_ms": 833.442, "count": 36, "max_ms": 16119.375, "p95_ms": 9969.823}, "triton_wait": {"avg_ms": 72.937, "count": 36, "max_ms": 76.919, "p95_ms": 76.45}, "visualize": {"avg_ms": 3.478, "count": 36, "max_ms": 4.195, "p95_ms": 3.604}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11687}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10443.82, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 172, "h264_decode_error_count": 177, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:07:53+09:00", "last_error_summary": "[h264 @ 0x56112f7a1380] error while decoding MB 77 32, bytestream -21 [in#0/rtsp @ 0x56112f07be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56112f07be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:07:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 391.013, "last_started_at": "2026-07-01T03:01:22+09:00", "pid": 74943, "process": {"exists": false, "pid": 74943}, "restart_count": 171, "restart_count_per_hour": 10, "rtsp_timeout_count": 280, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 172}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.601, "sample_count": 2588, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1402506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:07:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T03:07:53+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-01T03:07:53+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T03:07:53.248497+09:00", "mtime_epoch": 1782842873.248497, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90279}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:07:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1722.363, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 36, "pushed_frames": 36, "read_frames": 35, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.204, "pushed": 1.204, "read": 1.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.733, "count": 36, "max_ms": 20.231, "p95_ms": 20.118}, "gstreamer_push": {"avg_ms": 1.243, "count": 36, "max_ms": 1.403, "p95_ms": 1.326}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.106, "count": 36, "max_ms": 59.531, "p95_ms": 57.866}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.811, "count": 36, "max_ms": 67.917, "p95_ms": 67.311}, "yolo11_640_fire_11class": {"avg_ms": 66.979, "count": 36, "max_ms": 73.268, "p95_ms": 73.108}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.819, "count": 36, "max_ms": 76.907, "p95_ms": 76.438}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.876, "count": 36, "max_ms": 103.698, "p95_ms": 103.164}, "read_interval": {"avg_ms": 853.972, "count": 35, "max_ms": 16119.375, "p95_ms": 9969.823}, "triton_wait": {"avg_ms": 72.937, "count": 36, "max_ms": 76.919, "p95_ms": 76.45}, "visualize": {"avg_ms": 3.478, "count": 36, "max_ms": 4.195, "p95_ms": 3.604}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11687}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10628.336, "limit_mb": 8796093022207.996}, "detail": {"pid": 75095, "restart_count": 172, "start_count": 173}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 172, "h264_decode_error_count": 177, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:07:53+09:00", "last_error_summary": "[h264 @ 0x56112f7a1380] error while decoding MB 77 32, bytestream -21 [in#0/rtsp @ 0x56112f07be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56112f07be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:07:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:07:55+09:00", "pid": 75095, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.273, "name": "ffmpeg", "pid": 75095, "rss_mb": 209.273, "state": "S (sleeping)", "threads": 106}, "restart_count": 172, "restart_count_per_hour": 11, "rtsp_timeout_count": 280, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 173}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.616, "sample_count": 2544, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1402506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:07:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.073, "last_frame_pushed_at": "2026-07-01T03:07:53+09:00", "last_frame_read_age_sec": 4.172, "last_frame_read_at": "2026-07-01T03:07:53+09:00", "mjpeg": {"age_sec": 4.061, "exists": true, "mtime": "2026-07-01T03:07:53.356499+09:00", "mtime_epoch": 1782842873.3564992, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 85274}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:07:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1722.363, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.647, "pushed": 0.647, "read": 0.647}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.908, "count": 14, "max_ms": 20.231, "p95_ms": 20.231}, "gstreamer_push": {"avg_ms": 1.282, "count": 14, "max_ms": 1.403, "p95_ms": 1.403}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.214, "count": 14, "max_ms": 57.866, "p95_ms": 57.866}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.93, "count": 14, "max_ms": 67.311, "p95_ms": 67.311}, "yolo11_640_fire_11class": {"avg_ms": 65.564, "count": 14, "max_ms": 72.361, "p95_ms": 72.361}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.423, "count": 14, "max_ms": 73.756, "p95_ms": 73.756}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.773, "count": 14, "max_ms": 101.481, "p95_ms": 101.481}, "read_interval": {"avg_ms": 1960.027, "count": 14, "max_ms": 16119.375, "p95_ms": 16119.375}, "triton_wait": {"avg_ms": 70.574, "count": 14, "max_ms": 73.768, "p95_ms": 73.768}, "visualize": {"avg_ms": 3.5, "count": 14, "max_ms": 4.195, "p95_ms": 4.195}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.643, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11698}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10760.309, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 172, "h264_decode_error_count": 177, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:07:53+09:00", "last_error_summary": "[h264 @ 0x56112f7a1380] error while decoding MB 77 32, bytestream -21 [in#0/rtsp @ 0x56112f07be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56112f07be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:07:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 183.007, "last_started_at": "2026-07-01T03:07:55+09:00", "pid": 75095, "process": {"exists": true, "fd_count": 5, "hwm_mb": 327.844, "name": "ffmpeg", "pid": 75095, "rss_mb": 327.844, "state": "S (sleeping)", "threads": 106}, "restart_count": 172, "restart_count_per_hour": 10, "rtsp_timeout_count": 280, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 173}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.197, "sample_count": 2124, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1403863, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:10:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.057, "last_frame_pushed_at": "2026-07-01T03:10:28+09:00", "last_frame_read_age_sec": 30.139, "last_frame_read_at": "2026-07-01T03:10:28+09:00", "mjpeg": {"age_sec": 30.04, "exists": true, "mtime": "2026-07-01T03:10:28.383358+09:00", "mtime_epoch": 1782843028.3833578, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89870}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:10:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1723.137, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11698}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10754.32, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 172, "h264_decode_error_count": 177, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:07:53+09:00", "last_error_summary": "[h264 @ 0x56112f7a1380] error while decoding MB 77 32, bytestream -21 [in#0/rtsp @ 0x56112f07be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56112f07be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:07:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 183.019, "last_started_at": "2026-07-01T03:07:55+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 172, "restart_count_per_hour": 10, "rtsp_timeout_count": 280, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 173}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.197, "sample_count": 2124, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1403863, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:10:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.069, "last_frame_pushed_at": "2026-07-01T03:10:28+09:00", "last_frame_read_age_sec": 30.151, "last_frame_read_at": "2026-07-01T03:10:28+09:00", "mjpeg": {"age_sec": 30.052, "exists": true, "mtime": "2026-07-01T03:10:28.383358+09:00", "mtime_epoch": 1782843028.3833578, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89870}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:10:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1723.137, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11698}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10460.086, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a8d03e4180] error while decoding MB 42 21, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 173, "h264_decode_error_count": 178, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:10:58+09:00", "last_error_summary": "[h264 @ 0x55a8d03e4180] error while decoding MB 42 21, bytestream -9", "last_exit_at": "2026-07-01T03:07:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 183.07, "last_started_at": "2026-07-01T03:07:55+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 172, "restart_count_per_hour": 10, "rtsp_timeout_count": 280, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 173}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.197, "sample_count": 2123, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1403863, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:10:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.12, "last_frame_pushed_at": "2026-07-01T03:10:28+09:00", "last_frame_read_age_sec": 30.202, "last_frame_read_at": "2026-07-01T03:10:28+09:00", "mjpeg": {"age_sec": 30.103, "exists": true, "mtime": "2026-07-01T03:10:28.383358+09:00", "mtime_epoch": 1782843028.3833578, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89870}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:10:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1723.137, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11698}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10565.887, "limit_mb": 8796093022207.996}, "detail": {"pid": 75247, "restart_count": 173, "start_count": 174}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 173, "h264_decode_error_count": 178, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:10:58+09:00", "last_error_summary": "[h264 @ 0x55a8d03e4180] error while decoding MB 42 21, bytestream -9", "last_exit_at": "2026-07-01T03:07:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:11:01+09:00", "pid": 75247, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.832, "name": "ffmpeg", "pid": 75247, "rss_mb": 138.832, "state": "S (sleeping)", "threads": 19}, "restart_count": 173, "restart_count_per_hour": 11, "rtsp_timeout_count": 280, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 174}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.226, "sample_count": 2084, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1403863, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:10:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 34.649, "last_frame_pushed_at": "2026-07-01T03:10:28+09:00", "last_frame_read_age_sec": 34.732, "last_frame_read_at": "2026-07-01T03:10:28+09:00", "mjpeg": {"age_sec": 34.633, "exists": true, "mtime": "2026-07-01T03:10:28.383358+09:00", "mtime_epoch": 1782843028.3833578, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89870}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:11:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1723.137, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10557}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10491.781, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5605271f8d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605271f8d00] error while decoding MB 0 12, bytestream 8621 [h264 @ 0x560527a46b80] error while decoding MB 61 4, bytestream -5 [h264 @ 0x560527cb3580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560527cb3580] error while decoding MB 0 36, bytestream 3268 [h264 @ 0x5605270cbcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605270cbcc0] error while decoding MB 0 22, byte", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 368, "h264_decode_error_count": 1209, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:12:48+09:00", "last_error_summary": "[h264 @ 0x5605271f8d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605271f8d00] error while decoding MB 0 12, bytestream 8621 [h264 @ 0x560527a46b80] error while decoding MB 61 4, bytestream -5 [h264 @ 0x560527cb3580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560527cb3580] error while decoding MB 0 36, bytestream 3268 [h264 @ 0x5605270cbcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605270cbcc0] error while decoding MB 0 22, byte", "last_exit_at": "2026-07-01T03:01:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 692.711, "last_started_at": "2026-07-01T03:01:16+09:00", "pid": 74836, "process": {"exists": false, "pid": 74836}, "restart_count": 401, "restart_count_per_hour": 6, "rtsp_timeout_count": 354, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 402}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.673, "sample_count": 2705, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1266850, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:12:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-01T03:12:48+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T03:12:48+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-01T03:12:48.737932+09:00", "mtime_epoch": 1782843168.7379322, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:12:48+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4968.961, "name": "celery", "pid": 267, "rss_mb": 4968.961, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.703, "pushed": 3.703, "read": 3.703}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.296, "count": 111, "max_ms": 29.729, "p95_ms": 25.226}, "gstreamer_push": {"avg_ms": 1.24, "count": 111, "max_ms": 2.014, "p95_ms": 1.343}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.986, "count": 111, "max_ms": 47.823, "p95_ms": 34.477}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.483, "count": 111, "max_ms": 66.202, "p95_ms": 62.643}, "yolo11_640_fire_11class": {"avg_ms": 46.664, "count": 111, "max_ms": 67.479, "p95_ms": 63.489}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.66, "count": 111, "max_ms": 74.773, "p95_ms": 71.354}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.208, "count": 111, "max_ms": 104.549, "p95_ms": 100.627}, "read_interval": {"avg_ms": 270.328, "count": 111, "max_ms": 19487.371, "p95_ms": 115.434}, "triton_wait": {"avg_ms": 48.957, "count": 111, "max_ms": 74.785, "p95_ms": 71.366}, "visualize": {"avg_ms": 3.374, "count": 111, "max_ms": 3.834, "p95_ms": 3.707}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10557}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10491.152, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:12:48+09:00", "last_error_summary": "[h264 @ 0x5605271f8d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605271f8d00] error while decoding MB 0 12, bytestream 8621 [h264 @ 0x560527a46b80] error while decoding MB 61 4, bytestream -5 [h264 @ 0x560527cb3580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560527cb3580] error while decoding MB 0 36, bytestream 3268 [h264 @ 0x5605270cbcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605270cbcc0] error while decoding MB 0 22, byte", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 368, "h264_decode_error_count": 1209, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:12:48+09:00", "last_error_summary": "[h264 @ 0x5605271f8d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605271f8d00] error while decoding MB 0 12, bytestream 8621 [h264 @ 0x560527a46b80] error while decoding MB 61 4, bytestream -5 [h264 @ 0x560527cb3580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560527cb3580] error while decoding MB 0 36, bytestream 3268 [h264 @ 0x5605270cbcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605270cbcc0] error while decoding MB 0 22, byte", "last_exit_at": "2026-07-01T03:12:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 692.77, "last_started_at": "2026-07-01T03:01:16+09:00", "pid": 74836, "process": {"exists": false, "pid": 74836}, "restart_count": 401, "restart_count_per_hour": 6, "rtsp_timeout_count": 354, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 402}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.675, "sample_count": 2705, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1266851, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:12:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T03:12:48+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-01T03:12:48+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T03:12:48.737932+09:00", "mtime_epoch": 1782843168.7379322, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:12:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4968.961, "name": "celery", "pid": 267, "rss_mb": 4968.961, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.708, "pushed": 3.708, "read": 3.674}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.281, "count": 111, "max_ms": 29.729, "p95_ms": 25.226}, "gstreamer_push": {"avg_ms": 1.24, "count": 111, "max_ms": 2.014, "p95_ms": 1.343}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.0, "count": 111, "max_ms": 47.823, "p95_ms": 34.477}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.631, "count": 111, "max_ms": 68.515, "p95_ms": 62.71}, "yolo11_640_fire_11class": {"avg_ms": 46.835, "count": 111, "max_ms": 70.495, "p95_ms": 63.614}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.885, "count": 111, "max_ms": 77.915, "p95_ms": 71.614}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.413, "count": 111, "max_ms": 106.303, "p95_ms": 101.146}, "read_interval": {"avg_ms": 271.886, "count": 110, "max_ms": 19487.371, "p95_ms": 115.434}, "triton_wait": {"avg_ms": 49.181, "count": 111, "max_ms": 77.928, "p95_ms": 71.627}, "visualize": {"avg_ms": 3.371, "count": 111, "max_ms": 3.834, "p95_ms": 3.707}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10557}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10491.117, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 368, "h264_decode_error_count": 1209, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:12:48+09:00", "last_error_summary": "[h264 @ 0x5605271f8d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605271f8d00] error while decoding MB 0 12, bytestream 8621 [h264 @ 0x560527a46b80] error while decoding MB 61 4, bytestream -5 [h264 @ 0x560527cb3580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560527cb3580] error while decoding MB 0 36, bytestream 3268 [h264 @ 0x5605270cbcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605270cbcc0] error while decoding MB 0 22, byte", "last_exit_at": "2026-07-01T03:12:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 692.777, "last_started_at": "2026-07-01T03:01:16+09:00", "pid": 74836, "process": {"exists": false, "pid": 74836}, "restart_count": 401, "restart_count_per_hour": 6, "rtsp_timeout_count": 354, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 402}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.675, "sample_count": 2705, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1266851, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:12:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T03:12:48+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-01T03:12:48+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-01T03:12:48.737932+09:00", "mtime_epoch": 1782843168.7379322, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:12:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4968.961, "name": "celery", "pid": 267, "rss_mb": 4968.961, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.707, "pushed": 3.707, "read": 3.673}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.281, "count": 111, "max_ms": 29.729, "p95_ms": 25.226}, "gstreamer_push": {"avg_ms": 1.24, "count": 111, "max_ms": 2.014, "p95_ms": 1.343}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.0, "count": 111, "max_ms": 47.823, "p95_ms": 34.477}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.631, "count": 111, "max_ms": 68.515, "p95_ms": 62.71}, "yolo11_640_fire_11class": {"avg_ms": 46.835, "count": 111, "max_ms": 70.495, "p95_ms": 63.614}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.885, "count": 111, "max_ms": 77.915, "p95_ms": 71.614}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.413, "count": 111, "max_ms": 106.303, "p95_ms": 101.146}, "read_interval": {"avg_ms": 271.886, "count": 110, "max_ms": 19487.371, "p95_ms": 115.434}, "triton_wait": {"avg_ms": 49.181, "count": 111, "max_ms": 77.928, "p95_ms": 71.627}, "visualize": {"avg_ms": 3.371, "count": 111, "max_ms": 3.834, "p95_ms": 3.707}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10557}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10607.562, "limit_mb": 8796093022207.996}, "detail": {"pid": 75399, "restart_count": 402, "start_count": 403}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 368, "h264_decode_error_count": 1209, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:12:48+09:00", "last_error_summary": "[h264 @ 0x5605271f8d00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605271f8d00] error while decoding MB 0 12, bytestream 8621 [h264 @ 0x560527a46b80] error while decoding MB 61 4, bytestream -5 [h264 @ 0x560527cb3580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560527cb3580] error while decoding MB 0 36, bytestream 3268 [h264 @ 0x5605270cbcc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5605270cbcc0] error while decoding MB 0 22, byte", "last_exit_at": "2026-07-01T03:12:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:12:51+09:00", "pid": 75399, "process": {"exists": true, "fd_count": 5, "hwm_mb": 146.539, "name": "ffmpeg", "pid": 75399, "rss_mb": 146.539, "state": "S (sleeping)", "threads": 106}, "restart_count": 402, "restart_count_per_hour": 7, "rtsp_timeout_count": 354, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 403}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.727, "sample_count": 2659, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1266851, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:12:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.401, "last_frame_pushed_at": "2026-07-01T03:12:48+09:00", "last_frame_read_age_sec": 4.51, "last_frame_read_at": "2026-07-01T03:12:48+09:00", "mjpeg": {"age_sec": 4.385, "exists": true, "mtime": "2026-07-01T03:12:48.857934+09:00", "mtime_epoch": 1782843168.8579345, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120656}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:12:53+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4968.961, "name": "celery", "pid": 267, "rss_mb": 4968.961, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.136, "pushed": 2.136, "read": 2.102}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.616, "count": 64, "max_ms": 29.729, "p95_ms": 25.134}, "gstreamer_push": {"avg_ms": 1.223, "count": 64, "max_ms": 2.014, "p95_ms": 1.311}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.615, "count": 64, "max_ms": 47.823, "p95_ms": 34.495}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.779, "count": 64, "max_ms": 68.515, "p95_ms": 63.241}, "yolo11_640_fire_11class": {"avg_ms": 49.112, "count": 64, "max_ms": 70.495, "p95_ms": 64.877}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.63, "count": 64, "max_ms": 77.915, "p95_ms": 72.748}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.589, "count": 64, "max_ms": 106.303, "p95_ms": 102.606}, "read_interval": {"avg_ms": 405.324, "count": 63, "max_ms": 19487.371, "p95_ms": 117.25}, "triton_wait": {"avg_ms": 52.022, "count": 64, "max_ms": 77.928, "p95_ms": 72.76}, "visualize": {"avg_ms": 3.341, "count": 64, "max_ms": 3.707, "p95_ms": 3.564}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10493.625, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56066d48b880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56066d48b880] error while decoding MB 0 22, bytestream 1564 [h264 @ 0x56066d55a700] left block unavailable for requested intra mode [h264 @ 0x56066d55a700] error while decoding MB 0 15, bytestream 3337 [h264 @ 0x56066cb41d00] error while decoding MB 29 35, bytestream -5 [in#0/rtsp @ 0x56066ca01e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56066ca01e00] Error retrieving a packet from demuxer:", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 369, "h264_decode_error_count": 1212, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:16:10+09:00", "last_error_summary": "[h264 @ 0x56066d48b880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56066d48b880] error while decoding MB 0 22, bytestream 1564 [h264 @ 0x56066d55a700] left block unavailable for requested intra mode [h264 @ 0x56066d55a700] error while decoding MB 0 15, bytestream 3337 [h264 @ 0x56066cb41d00] error while decoding MB 29 35, bytestream -5 [in#0/rtsp @ 0x56066ca01e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56066ca01e00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-01T03:12:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.41, "last_started_at": "2026-07-01T03:12:51+09:00", "pid": 75399, "process": {"exists": false, "pid": 75399}, "restart_count": 402, "restart_count_per_hour": 7, "rtsp_timeout_count": 356, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 403}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.66, "sample_count": 2321, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1268392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:16:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-01T03:16:10+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T03:16:10+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-01T03:16:10.601618+09:00", "mtime_epoch": 1782843370.601618, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118216}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:16:10+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4969.477, "name": "celery", "pid": 267, "rss_mb": 4969.477, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 95, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.17, "pushed": 3.17, "read": 3.17}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.648, "count": 95, "max_ms": 28.467, "p95_ms": 23.15}, "gstreamer_push": {"avg_ms": 1.209, "count": 95, "max_ms": 1.575, "p95_ms": 1.285}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.241, "count": 95, "max_ms": 43.79, "p95_ms": 42.522}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.435, "count": 95, "max_ms": 67.608, "p95_ms": 66.216}, "yolo11_640_fire_11class": {"avg_ms": 63.151, "count": 95, "max_ms": 69.109, "p95_ms": 67.808}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.75, "count": 95, "max_ms": 76.854, "p95_ms": 76.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.72, "count": 95, "max_ms": 107.113, "p95_ms": 105.955}, "read_interval": {"avg_ms": 315.972, "count": 95, "max_ms": 10130.21, "p95_ms": 121.268}, "triton_wait": {"avg_ms": 70.828, "count": 95, "max_ms": 76.866, "p95_ms": 76.043}, "visualize": {"avg_ms": 3.395, "count": 95, "max_ms": 4.547, "p95_ms": 3.649}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10492.938, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:16:10+09:00", "last_error_summary": "[h264 @ 0x56066d48b880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56066d48b880] error while decoding MB 0 22, bytestream 1564 [h264 @ 0x56066d55a700] left block unavailable for requested intra mode [h264 @ 0x56066d55a700] error while decoding MB 0 15, bytestream 3337 [h264 @ 0x56066cb41d00] error while decoding MB 29 35, bytestream -5 [in#0/rtsp @ 0x56066ca01e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56066ca01e00] Error retrieving a packet from demuxer:", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 369, "h264_decode_error_count": 1212, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:16:10+09:00", "last_error_summary": "[h264 @ 0x56066d48b880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56066d48b880] error while decoding MB 0 22, bytestream 1564 [h264 @ 0x56066d55a700] left block unavailable for requested intra mode [h264 @ 0x56066d55a700] error while decoding MB 0 15, bytestream 3337 [h264 @ 0x56066cb41d00] error while decoding MB 29 35, bytestream -5 [in#0/rtsp @ 0x56066ca01e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56066ca01e00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-01T03:16:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.472, "last_started_at": "2026-07-01T03:12:51+09:00", "pid": 75399, "process": {"exists": false, "pid": 75399}, "restart_count": 402, "restart_count_per_hour": 7, "rtsp_timeout_count": 356, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 403}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.666, "sample_count": 2322, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1268393, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:16:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T03:16:10+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-01T03:16:10+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-01T03:16:10.601618+09:00", "mtime_epoch": 1782843370.601618, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118216}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:16:10+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4969.477, "name": "celery", "pid": 267, "rss_mb": 4969.477, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.175, "pushed": 3.175, "read": 3.142}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.636, "count": 95, "max_ms": 28.467, "p95_ms": 23.15}, "gstreamer_push": {"avg_ms": 1.21, "count": 95, "max_ms": 1.575, "p95_ms": 1.285}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.105, "count": 95, "max_ms": 42.885, "p95_ms": 42.337}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.518, "count": 95, "max_ms": 69.371, "p95_ms": 66.73}, "yolo11_640_fire_11class": {"avg_ms": 63.246, "count": 95, "max_ms": 71.765, "p95_ms": 68.134}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.848, "count": 95, "max_ms": 79.367, "p95_ms": 76.095}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.802, "count": 95, "max_ms": 108.833, "p95_ms": 106.24}, "read_interval": {"avg_ms": 318.1, "count": 94, "max_ms": 10130.21, "p95_ms": 121.268}, "triton_wait": {"avg_ms": 70.926, "count": 95, "max_ms": 79.38, "p95_ms": 76.107}, "visualize": {"avg_ms": 3.393, "count": 95, "max_ms": 4.547, "p95_ms": 3.649}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10492.91, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 369, "h264_decode_error_count": 1212, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:16:10+09:00", "last_error_summary": "[h264 @ 0x56066d48b880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56066d48b880] error while decoding MB 0 22, bytestream 1564 [h264 @ 0x56066d55a700] left block unavailable for requested intra mode [h264 @ 0x56066d55a700] error while decoding MB 0 15, bytestream 3337 [h264 @ 0x56066cb41d00] error while decoding MB 29 35, bytestream -5 [in#0/rtsp @ 0x56066ca01e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56066ca01e00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-01T03:16:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.479, "last_started_at": "2026-07-01T03:12:51+09:00", "pid": 75399, "process": {"exists": false, "pid": 75399}, "restart_count": 402, "restart_count_per_hour": 7, "rtsp_timeout_count": 356, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 403}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.666, "sample_count": 2322, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1268393, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:16:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T03:16:10+09:00", "last_frame_read_age_sec": 0.123, "last_frame_read_at": "2026-07-01T03:16:10+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-01T03:16:10.601618+09:00", "mtime_epoch": 1782843370.601618, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118216}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:16:10+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4969.477, "name": "celery", "pid": 267, "rss_mb": 4969.477, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.175, "pushed": 3.175, "read": 3.141}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.636, "count": 95, "max_ms": 28.467, "p95_ms": 23.15}, "gstreamer_push": {"avg_ms": 1.21, "count": 95, "max_ms": 1.575, "p95_ms": 1.285}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.105, "count": 95, "max_ms": 42.885, "p95_ms": 42.337}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.518, "count": 95, "max_ms": 69.371, "p95_ms": 66.73}, "yolo11_640_fire_11class": {"avg_ms": 63.246, "count": 95, "max_ms": 71.765, "p95_ms": 68.134}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.848, "count": 95, "max_ms": 79.367, "p95_ms": 76.095}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.802, "count": 95, "max_ms": 108.833, "p95_ms": 106.24}, "read_interval": {"avg_ms": 318.1, "count": 94, "max_ms": 10130.21, "p95_ms": 121.268}, "triton_wait": {"avg_ms": 70.926, "count": 95, "max_ms": 79.38, "p95_ms": 76.107}, "visualize": {"avg_ms": 3.393, "count": 95, "max_ms": 4.547, "p95_ms": 3.649}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10601.434, "limit_mb": 8796093022207.996}, "detail": {"pid": 75506, "restart_count": 403, "start_count": 404}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 369, "h264_decode_error_count": 1212, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:16:10+09:00", "last_error_summary": "[h264 @ 0x56066d48b880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56066d48b880] error while decoding MB 0 22, bytestream 1564 [h264 @ 0x56066d55a700] left block unavailable for requested intra mode [h264 @ 0x56066d55a700] error while decoding MB 0 15, bytestream 3337 [h264 @ 0x56066cb41d00] error while decoding MB 29 35, bytestream -5 [in#0/rtsp @ 0x56066ca01e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56066ca01e00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-01T03:16:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:16:13+09:00", "pid": 75506, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.746, "name": "ffmpeg", "pid": 75506, "rss_mb": 138.746, "state": "S (sleeping)", "threads": 106}, "restart_count": 403, "restart_count_per_hour": 8, "rtsp_timeout_count": 356, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 404}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.646, "sample_count": 2278, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1268393, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:16:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.992, "last_frame_pushed_at": "2026-07-01T03:16:10+09:00", "last_frame_read_age_sec": 5.103, "last_frame_read_at": "2026-07-01T03:16:10+09:00", "mjpeg": {"age_sec": 4.975, "exists": true, "mtime": "2026-07-01T03:16:10.725620+09:00", "mtime_epoch": 1782843370.7256203, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119925}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:16:15+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4969.477, "name": "celery", "pid": 267, "rss_mb": 4969.477, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 51, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.738, "pushed": 1.738, "read": 1.705}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.498, "count": 52, "max_ms": 23.624, "p95_ms": 23.14}, "gstreamer_push": {"avg_ms": 1.213, "count": 52, "max_ms": 1.575, "p95_ms": 1.288}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.764, "count": 52, "max_ms": 42.885, "p95_ms": 42.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.618, "count": 52, "max_ms": 69.371, "p95_ms": 67.054}, "yolo11_640_fire_11class": {"avg_ms": 62.82, "count": 52, "max_ms": 71.765, "p95_ms": 68.651}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.066, "count": 52, "max_ms": 79.367, "p95_ms": 76.126}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.815, "count": 52, "max_ms": 108.833, "p95_ms": 106.379}, "read_interval": {"avg_ms": 488.609, "count": 51, "max_ms": 10130.21, "p95_ms": 121.418}, "triton_wait": {"avg_ms": 71.079, "count": 52, "max_ms": 79.38, "p95_ms": 76.138}, "visualize": {"avg_ms": 3.391, "count": 52, "max_ms": 4.111, "p95_ms": 3.818}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11723}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10429.844, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5598c2d8c380] error while decoding MB 77 28, bytestream -5 [in#0/rtsp @ 0x5598c2d49e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5598c2d49e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 174, "h264_decode_error_count": 179, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:16:55+09:00", "last_error_summary": "[h264 @ 0x5598c2d8c380] error while decoding MB 77 28, bytestream -5 [in#0/rtsp @ 0x5598c2d49e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5598c2d49e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:07:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 354.623, "last_started_at": "2026-07-01T03:11:01+09:00", "pid": 75247, "process": {"exists": false, "pid": 75247}, "restart_count": 173, "restart_count_per_hour": 9, "rtsp_timeout_count": 282, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 174}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.587, "sample_count": 2454, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1406817, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:16:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-01T03:16:55+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-01T03:16:55+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T03:16:55.578437+09:00", "mtime_epoch": 1782843415.578437, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89151}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:16:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1724.426, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.714, "pushed": 0.714, "read": 0.769}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.135, "count": 13, "max_ms": 20.254, "p95_ms": 20.254}, "gstreamer_push": {"avg_ms": 1.24, "count": 13, "max_ms": 1.339, "p95_ms": 1.339}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.043, "count": 13, "max_ms": 56.013, "p95_ms": 56.013}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.682, "count": 13, "max_ms": 66.025, "p95_ms": 66.025}, "yolo11_640_fire_11class": {"avg_ms": 60.769, "count": 13, "max_ms": 68.302, "p95_ms": 68.302}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.926, "count": 13, "max_ms": 74.526, "p95_ms": 74.526}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.049, "count": 13, "max_ms": 100.795, "p95_ms": 100.795}, "read_interval": {"avg_ms": 2723.985, "count": 14, "max_ms": 19995.381, "p95_ms": 19995.381}, "triton_wait": {"avg_ms": 63.834, "count": 13, "max_ms": 74.538, "p95_ms": 74.538}, "visualize": {"avg_ms": 3.397, "count": 13, "max_ms": 3.48, "p95_ms": 3.48}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.206, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11723}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10429.492, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:16:55+09:00", "last_error_summary": "[h264 @ 0x5598c2d8c380] error while decoding MB 77 28, bytestream -5 [in#0/rtsp @ 0x5598c2d49e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5598c2d49e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 174, "h264_decode_error_count": 179, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:16:55+09:00", "last_error_summary": "[h264 @ 0x5598c2d8c380] error while decoding MB 77 28, bytestream -5 [in#0/rtsp @ 0x5598c2d49e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5598c2d49e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:16:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 354.665, "last_started_at": "2026-07-01T03:11:01+09:00", "pid": 75247, "process": {"exists": false, "pid": 75247}, "restart_count": 173, "restart_count_per_hour": 9, "rtsp_timeout_count": 282, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 174}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.587, "sample_count": 2455, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1406818, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:16:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T03:16:55+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-01T03:16:55+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-01T03:16:55.578437+09:00", "mtime_epoch": 1782843415.578437, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89151}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:16:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1724.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.767, "pushed": 0.767, "read": 0.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.215, "count": 14, "max_ms": 20.254, "p95_ms": 20.254}, "gstreamer_push": {"avg_ms": 1.243, "count": 14, "max_ms": 1.339, "p95_ms": 1.339}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.953, "count": 14, "max_ms": 56.013, "p95_ms": 56.013}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.442, "count": 14, "max_ms": 69.327, "p95_ms": 69.327}, "yolo11_640_fire_11class": {"avg_ms": 61.381, "count": 14, "max_ms": 69.338, "p95_ms": 69.338}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.56, "count": 14, "max_ms": 74.526, "p95_ms": 74.526}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.769, "count": 14, "max_ms": 100.795, "p95_ms": 100.795}, "read_interval": {"avg_ms": 2723.985, "count": 14, "max_ms": 19995.381, "p95_ms": 19995.381}, "triton_wait": {"avg_ms": 64.404, "count": 14, "max_ms": 74.538, "p95_ms": 74.538}, "visualize": {"avg_ms": 3.46, "count": 14, "max_ms": 4.278, "p95_ms": 4.278}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.248, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11723}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10429.477, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 174, "h264_decode_error_count": 179, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:16:55+09:00", "last_error_summary": "[h264 @ 0x5598c2d8c380] error while decoding MB 77 28, bytestream -5 [in#0/rtsp @ 0x5598c2d49e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5598c2d49e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:16:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 354.672, "last_started_at": "2026-07-01T03:11:01+09:00", "pid": 75247, "process": {"exists": false, "pid": 75247}, "restart_count": 173, "restart_count_per_hour": 9, "rtsp_timeout_count": 282, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 174}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.587, "sample_count": 2455, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1406818, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:16:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T03:16:55+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-01T03:16:55+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-01T03:16:55.578437+09:00", "mtime_epoch": 1782843415.578437, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89151}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:16:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1724.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.767, "pushed": 0.767, "read": 0.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.215, "count": 14, "max_ms": 20.254, "p95_ms": 20.254}, "gstreamer_push": {"avg_ms": 1.243, "count": 14, "max_ms": 1.339, "p95_ms": 1.339}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.953, "count": 14, "max_ms": 56.013, "p95_ms": 56.013}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.442, "count": 14, "max_ms": 69.327, "p95_ms": 69.327}, "yolo11_640_fire_11class": {"avg_ms": 61.381, "count": 14, "max_ms": 69.338, "p95_ms": 69.338}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.56, "count": 14, "max_ms": 74.526, "p95_ms": 74.526}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.769, "count": 14, "max_ms": 100.795, "p95_ms": 100.795}, "read_interval": {"avg_ms": 2723.985, "count": 14, "max_ms": 19995.381, "p95_ms": 19995.381}, "triton_wait": {"avg_ms": 64.404, "count": 14, "max_ms": 74.538, "p95_ms": 74.538}, "visualize": {"avg_ms": 3.46, "count": 14, "max_ms": 4.278, "p95_ms": 4.278}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.254, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11723}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10553.113, "limit_mb": 8796093022207.996}, "detail": {"pid": 75613, "restart_count": 174, "start_count": 175}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 174, "h264_decode_error_count": 179, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:16:55+09:00", "last_error_summary": "[h264 @ 0x5598c2d8c380] error while decoding MB 77 28, bytestream -5 [in#0/rtsp @ 0x5598c2d49e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5598c2d49e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:16:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:16:58+09:00", "pid": 75613, "process": {"exists": true, "fd_count": 5, "hwm_mb": 155.984, "name": "ffmpeg", "pid": 75613, "rss_mb": 155.82, "state": "R (running)", "threads": 49}, "restart_count": 174, "restart_count_per_hour": 10, "rtsp_timeout_count": 282, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 175}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.621, "sample_count": 2406, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1406818, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:16:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.527, "last_frame_pushed_at": "2026-07-01T03:16:55+09:00", "last_frame_read_age_sec": 4.63, "last_frame_read_at": "2026-07-01T03:16:55+09:00", "mjpeg": {"age_sec": 4.512, "exists": true, "mtime": "2026-07-01T03:16:55.690439+09:00", "mtime_epoch": 1782843415.6904392, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 88924}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:17:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1724.426, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.615, "pushed": 0.615, "read": 0.615}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.215, "count": 14, "max_ms": 20.254, "p95_ms": 20.254}, "gstreamer_push": {"avg_ms": 1.243, "count": 14, "max_ms": 1.339, "p95_ms": 1.339}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.953, "count": 14, "max_ms": 56.013, "p95_ms": 56.013}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.442, "count": 14, "max_ms": 69.327, "p95_ms": 69.327}, "yolo11_640_fire_11class": {"avg_ms": 61.381, "count": 14, "max_ms": 69.338, "p95_ms": 69.338}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.56, "count": 14, "max_ms": 74.526, "p95_ms": 74.526}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.769, "count": 14, "max_ms": 100.795, "p95_ms": 100.795}, "read_interval": {"avg_ms": 2723.985, "count": 14, "max_ms": 19995.381, "p95_ms": 19995.381}, "triton_wait": {"avg_ms": 64.404, "count": 14, "max_ms": 74.538, "p95_ms": 74.538}, "visualize": {"avg_ms": 3.46, "count": 14, "max_ms": 4.278, "p95_ms": 4.278}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.77, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11749}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10450.234, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55fc0151c700] error while decoding MB 27 21, bytestream -11 [in#0/rtsp @ 0x55fc00abee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc00abee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 175, "h264_decode_error_count": 180, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:23:03+09:00", "last_error_summary": "[h264 @ 0x55fc0151c700] error while decoding MB 27 21, bytestream -11 [in#0/rtsp @ 0x55fc00abee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc00abee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:16:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 364.896, "last_started_at": "2026-07-01T03:16:58+09:00", "pid": 75613, "process": {"exists": false, "pid": 75613}, "restart_count": 174, "restart_count_per_hour": 9, "rtsp_timeout_count": 284, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 175}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.104, "sample_count": 2551, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1409916, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:23:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-01T03:23:03+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-01T03:23:03+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T03:23:03.037110+09:00", "mtime_epoch": 1782843783.0371103, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92951}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:23:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1725.973, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 50, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.67, "pushed": 1.67, "read": 1.67}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.394, "count": 50, "max_ms": 21.243, "p95_ms": 19.699}, "gstreamer_push": {"avg_ms": 1.227, "count": 50, "max_ms": 1.324, "p95_ms": 1.298}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.595, "count": 50, "max_ms": 62.024, "p95_ms": 56.655}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.732, "count": 50, "max_ms": 70.867, "p95_ms": 67.388}, "yolo11_640_fire_11class": {"avg_ms": 58.8, "count": 50, "max_ms": 70.485, "p95_ms": 69.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.058, "count": 50, "max_ms": 76.794, "p95_ms": 75.627}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.895, "count": 50, "max_ms": 103.559, "p95_ms": 102.193}, "read_interval": {"avg_ms": 599.008, "count": 50, "max_ms": 14674.337, "p95_ms": 123.006}, "triton_wait": {"avg_ms": 62.252, "count": 50, "max_ms": 76.807, "p95_ms": 75.64}, "visualize": {"avg_ms": 3.451, "count": 50, "max_ms": 3.713, "p95_ms": 3.639}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11749}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10449.773, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:23:03+09:00", "last_error_summary": "[h264 @ 0x55fc0151c700] error while decoding MB 27 21, bytestream -11 [in#0/rtsp @ 0x55fc00abee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc00abee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 175, "h264_decode_error_count": 180, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:23:03+09:00", "last_error_summary": "[h264 @ 0x55fc0151c700] error while decoding MB 27 21, bytestream -11 [in#0/rtsp @ 0x55fc00abee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc00abee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:23:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 364.947, "last_started_at": "2026-07-01T03:16:58+09:00", "pid": 75613, "process": {"exists": false, "pid": 75613}, "restart_count": 174, "restart_count_per_hour": 9, "rtsp_timeout_count": 284, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 175}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.104, "sample_count": 2552, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1409917, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:23:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T03:23:03+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-01T03:23:03+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-01T03:23:03.037110+09:00", "mtime_epoch": 1782843783.0371103, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92951}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:23:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1725.973, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.701, "pushed": 1.701, "read": 1.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.394, "count": 51, "max_ms": 21.243, "p95_ms": 19.699}, "gstreamer_push": {"avg_ms": 1.227, "count": 51, "max_ms": 1.324, "p95_ms": 1.298}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.555, "count": 51, "max_ms": 62.024, "p95_ms": 56.655}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.981, "count": 51, "max_ms": 70.867, "p95_ms": 68.213}, "yolo11_640_fire_11class": {"avg_ms": 59.023, "count": 51, "max_ms": 70.485, "p95_ms": 70.051}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.36, "count": 51, "max_ms": 76.794, "p95_ms": 76.363}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.173, "count": 51, "max_ms": 103.559, "p95_ms": 102.68}, "read_interval": {"avg_ms": 599.008, "count": 50, "max_ms": 14674.337, "p95_ms": 123.006}, "triton_wait": {"avg_ms": 62.531, "count": 51, "max_ms": 76.807, "p95_ms": 76.376}, "visualize": {"avg_ms": 3.449, "count": 51, "max_ms": 3.713, "p95_ms": 3.639}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11749}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10449.906, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 175, "h264_decode_error_count": 180, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:23:03+09:00", "last_error_summary": "[h264 @ 0x55fc0151c700] error while decoding MB 27 21, bytestream -11 [in#0/rtsp @ 0x55fc00abee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc00abee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:23:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 364.956, "last_started_at": "2026-07-01T03:16:58+09:00", "pid": 75613, "process": {"exists": false, "pid": 75613}, "restart_count": 174, "restart_count_per_hour": 9, "rtsp_timeout_count": 284, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 175}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.104, "sample_count": 2552, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1409917, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:23:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-01T03:23:03+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-01T03:23:03+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-01T03:23:03.037110+09:00", "mtime_epoch": 1782843783.0371103, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92951}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:23:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1725.973, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.7, "pushed": 1.7, "read": 1.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.394, "count": 51, "max_ms": 21.243, "p95_ms": 19.699}, "gstreamer_push": {"avg_ms": 1.227, "count": 51, "max_ms": 1.324, "p95_ms": 1.298}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.555, "count": 51, "max_ms": 62.024, "p95_ms": 56.655}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.981, "count": 51, "max_ms": 70.867, "p95_ms": 68.213}, "yolo11_640_fire_11class": {"avg_ms": 59.023, "count": 51, "max_ms": 70.485, "p95_ms": 70.051}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.36, "count": 51, "max_ms": 76.794, "p95_ms": 76.363}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.173, "count": 51, "max_ms": 103.559, "p95_ms": 102.68}, "read_interval": {"avg_ms": 599.008, "count": 50, "max_ms": 14674.337, "p95_ms": 123.006}, "triton_wait": {"avg_ms": 62.531, "count": 51, "max_ms": 76.807, "p95_ms": 76.376}, "visualize": {"avg_ms": 3.449, "count": 51, "max_ms": 3.713, "p95_ms": 3.639}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11749}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10629.582, "limit_mb": 8796093022207.996}, "detail": {"pid": 75765, "restart_count": 175, "start_count": 176}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 175, "h264_decode_error_count": 180, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:23:03+09:00", "last_error_summary": "[h264 @ 0x55fc0151c700] error while decoding MB 27 21, bytestream -11 [in#0/rtsp @ 0x55fc00abee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc00abee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:23:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:23:05+09:00", "pid": 75765, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.309, "name": "ffmpeg", "pid": 75765, "rss_mb": 209.309, "state": "S (sleeping)", "threads": 106}, "restart_count": 175, "restart_count_per_hour": 10, "rtsp_timeout_count": 284, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 176}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.076, "sample_count": 2514, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1409917, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:23:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.474, "last_frame_pushed_at": "2026-07-01T03:23:03+09:00", "last_frame_read_age_sec": 4.579, "last_frame_read_at": "2026-07-01T03:23:03+09:00", "mjpeg": {"age_sec": 4.458, "exists": true, "mtime": "2026-07-01T03:23:03.157113+09:00", "mtime_epoch": 1782843783.1571126, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93141}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:23:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1725.973, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.812, "pushed": 0.812, "read": 0.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.566, "count": 13, "max_ms": 21.243, "p95_ms": 21.243}, "gstreamer_push": {"avg_ms": 1.229, "count": 13, "max_ms": 1.304, "p95_ms": 1.304}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.577, "count": 13, "max_ms": 59.584, "p95_ms": 59.584}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.138, "count": 13, "max_ms": 69.444, "p95_ms": 69.444}, "yolo11_640_fire_11class": {"avg_ms": 65.401, "count": 13, "max_ms": 70.157, "p95_ms": 70.157}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.457, "count": 13, "max_ms": 76.482, "p95_ms": 76.482}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.346, "count": 13, "max_ms": 103.09, "p95_ms": 103.09}, "read_interval": {"avg_ms": 2007.374, "count": 13, "max_ms": 14674.337, "p95_ms": 14674.337}, "triton_wait": {"avg_ms": 71.605, "count": 13, "max_ms": 76.494, "p95_ms": 76.494}, "visualize": {"avg_ms": 3.445, "count": 13, "max_ms": 3.713, "p95_ms": 3.713}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.001, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10628}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10499.867, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d550d809c0] top block unavailable for requested intra mode -1 [h264 @ 0x55d550d809c0] error while decoding MB 12 0, bytestream 4944 [h264 @ 0x55d550c3b740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d550c3b740] error while decoding MB 0 7, bytestream 1825 [h264 @ 0x55d551769580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d551769580] error while decoding MB 0 25, bytestream 58556 [h264 @ 0x55d5515cba00] reference picture missing during reo", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 370, "h264_decode_error_count": 1216, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:28:50+09:00", "last_error_summary": "[h264 @ 0x55d550d809c0] top block unavailable for requested intra mode -1 [h264 @ 0x55d550d809c0] error while decoding MB 12 0, bytestream 4944 [h264 @ 0x55d550c3b740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d550c3b740] error while decoding MB 0 7, bytestream 1825 [h264 @ 0x55d551769580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d551769580] error while decoding MB 0 25, bytestream 58556 [h264 @ 0x55d5515cba00] reference picture missing during reo", "last_exit_at": "2026-07-01T03:16:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 757.224, "last_started_at": "2026-07-01T03:16:13+09:00", "pid": 75506, "process": {"exists": false, "pid": 75506}, "restart_count": 403, "restart_count_per_hour": 7, "rtsp_timeout_count": 358, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 404}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.942, "sample_count": 2691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1275389, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:28:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.082, "last_frame_pushed_at": "2026-07-01T03:28:50+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T03:28:50+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-01T03:28:50.859407+09:00", "mtime_epoch": 1782844130.8594072, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 131765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:28:50+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4972.828, "name": "celery", "pid": 267, "rss_mb": 4972.828, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 91, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.041, "pushed": 3.041, "read": 3.041}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.671, "count": 91, "max_ms": 34.949, "p95_ms": 34.08}, "gstreamer_push": {"avg_ms": 1.197, "count": 91, "max_ms": 1.431, "p95_ms": 1.342}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.652, "count": 91, "max_ms": 44.652, "p95_ms": 42.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.508, "count": 91, "max_ms": 68.862, "p95_ms": 67.855}, "yolo11_640_fire_11class": {"avg_ms": 64.195, "count": 91, "max_ms": 70.593, "p95_ms": 68.979}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.751, "count": 91, "max_ms": 78.008, "p95_ms": 77.433}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.037, "count": 91, "max_ms": 118.888, "p95_ms": 117.548}, "read_interval": {"avg_ms": 329.359, "count": 91, "max_ms": 19192.261, "p95_ms": 135.134}, "triton_wait": {"avg_ms": 72.763, "count": 91, "max_ms": 78.02, "p95_ms": 77.444}, "visualize": {"avg_ms": 3.743, "count": 91, "max_ms": 5.725, "p95_ms": 5.443}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10628}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10499.691, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:28:50+09:00", "last_error_summary": "[h264 @ 0x55d550d809c0] top block unavailable for requested intra mode -1 [h264 @ 0x55d550d809c0] error while decoding MB 12 0, bytestream 4944 [h264 @ 0x55d550c3b740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d550c3b740] error while decoding MB 0 7, bytestream 1825 [h264 @ 0x55d551769580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d551769580] error while decoding MB 0 25, bytestream 58556 [h264 @ 0x55d5515cba00] reference picture missing during reo", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 370, "h264_decode_error_count": 1216, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:28:50+09:00", "last_error_summary": "[h264 @ 0x55d550d809c0] top block unavailable for requested intra mode -1 [h264 @ 0x55d550d809c0] error while decoding MB 12 0, bytestream 4944 [h264 @ 0x55d550c3b740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d550c3b740] error while decoding MB 0 7, bytestream 1825 [h264 @ 0x55d551769580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d551769580] error while decoding MB 0 25, bytestream 58556 [h264 @ 0x55d5515cba00] reference picture missing during reo", "last_exit_at": "2026-07-01T03:28:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 757.266, "last_started_at": "2026-07-01T03:16:13+09:00", "pid": 75506, "process": {"exists": false, "pid": 75506}, "restart_count": 403, "restart_count_per_hour": 7, "rtsp_timeout_count": 358, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 404}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.937, "sample_count": 2692, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1275390, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:28:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T03:28:50+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T03:28:50+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-01T03:28:50.859407+09:00", "mtime_epoch": 1782844130.8594072, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 131765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:28:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4972.828, "name": "celery", "pid": 267, "rss_mb": 4972.828, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 91, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.07, "pushed": 3.07, "read": 3.037}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.644, "count": 92, "max_ms": 34.949, "p95_ms": 34.08}, "gstreamer_push": {"avg_ms": 1.197, "count": 92, "max_ms": 1.431, "p95_ms": 1.342}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.629, "count": 92, "max_ms": 44.652, "p95_ms": 42.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.548, "count": 92, "max_ms": 68.862, "p95_ms": 67.855}, "yolo11_640_fire_11class": {"avg_ms": 64.22, "count": 92, "max_ms": 70.593, "p95_ms": 68.979}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.784, "count": 92, "max_ms": 78.008, "p95_ms": 77.433}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.037, "count": 92, "max_ms": 118.888, "p95_ms": 117.548}, "read_interval": {"avg_ms": 329.359, "count": 91, "max_ms": 19192.261, "p95_ms": 135.134}, "triton_wait": {"avg_ms": 72.796, "count": 92, "max_ms": 78.02, "p95_ms": 77.444}, "visualize": {"avg_ms": 3.741, "count": 92, "max_ms": 5.725, "p95_ms": 5.443}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10628}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10499.523, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 370, "h264_decode_error_count": 1216, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:28:50+09:00", "last_error_summary": "[h264 @ 0x55d550d809c0] top block unavailable for requested intra mode -1 [h264 @ 0x55d550d809c0] error while decoding MB 12 0, bytestream 4944 [h264 @ 0x55d550c3b740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d550c3b740] error while decoding MB 0 7, bytestream 1825 [h264 @ 0x55d551769580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d551769580] error while decoding MB 0 25, bytestream 58556 [h264 @ 0x55d5515cba00] reference picture missing during reo", "last_exit_at": "2026-07-01T03:28:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 757.274, "last_started_at": "2026-07-01T03:16:13+09:00", "pid": 75506, "process": {"exists": false, "pid": 75506}, "restart_count": 403, "restart_count_per_hour": 7, "rtsp_timeout_count": 358, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 404}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.937, "sample_count": 2692, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1275390, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:28:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T03:28:50+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-01T03:28:50+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-01T03:28:50.859407+09:00", "mtime_epoch": 1782844130.8594072, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 131765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:28:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4972.828, "name": "celery", "pid": 267, "rss_mb": 4972.828, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 91, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.069, "pushed": 3.069, "read": 3.036}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.644, "count": 92, "max_ms": 34.949, "p95_ms": 34.08}, "gstreamer_push": {"avg_ms": 1.197, "count": 92, "max_ms": 1.431, "p95_ms": 1.342}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.629, "count": 92, "max_ms": 44.652, "p95_ms": 42.189}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.548, "count": 92, "max_ms": 68.862, "p95_ms": 67.855}, "yolo11_640_fire_11class": {"avg_ms": 64.22, "count": 92, "max_ms": 70.593, "p95_ms": 68.979}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.784, "count": 92, "max_ms": 78.008, "p95_ms": 77.433}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.037, "count": 92, "max_ms": 118.888, "p95_ms": 117.548}, "read_interval": {"avg_ms": 329.359, "count": 91, "max_ms": 19192.261, "p95_ms": 135.134}, "triton_wait": {"avg_ms": 72.796, "count": 92, "max_ms": 78.02, "p95_ms": 77.444}, "visualize": {"avg_ms": 3.741, "count": 92, "max_ms": 5.725, "p95_ms": 5.443}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10628}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10608.172, "limit_mb": 8796093022207.996}, "detail": {"pid": 75917, "restart_count": 404, "start_count": 405}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 370, "h264_decode_error_count": 1216, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:28:50+09:00", "last_error_summary": "[h264 @ 0x55d550d809c0] top block unavailable for requested intra mode -1 [h264 @ 0x55d550d809c0] error while decoding MB 12 0, bytestream 4944 [h264 @ 0x55d550c3b740] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d550c3b740] error while decoding MB 0 7, bytestream 1825 [h264 @ 0x55d551769580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d551769580] error while decoding MB 0 25, bytestream 58556 [h264 @ 0x55d5515cba00] reference picture missing during reo", "last_exit_at": "2026-07-01T03:28:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:28:52+09:00", "pid": 75917, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.125, "name": "ffmpeg", "pid": 75917, "rss_mb": 139.125, "state": "S (sleeping)", "threads": 106}, "restart_count": 404, "restart_count_per_hour": 8, "rtsp_timeout_count": 358, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 405}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.865, "sample_count": 2658, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1275390, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:28:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.037, "last_frame_pushed_at": "2026-07-01T03:28:50+09:00", "last_frame_read_age_sec": 4.143, "last_frame_read_at": "2026-07-01T03:28:50+09:00", "mjpeg": {"age_sec": 4.013, "exists": true, "mtime": "2026-07-01T03:28:50.983410+09:00", "mtime_epoch": 1782844130.9834096, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129404}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:28:55+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4972.828, "name": "celery", "pid": 267, "rss_mb": 4972.828, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.967, "pushed": 1.967, "read": 1.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.347, "count": 59, "max_ms": 23.414, "p95_ms": 23.299}, "gstreamer_push": {"avg_ms": 1.212, "count": 59, "max_ms": 1.431, "p95_ms": 1.389}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.602, "count": 59, "max_ms": 44.652, "p95_ms": 41.851}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.029, "count": 59, "max_ms": 68.862, "p95_ms": 68.361}, "yolo11_640_fire_11class": {"avg_ms": 64.705, "count": 59, "max_ms": 70.593, "p95_ms": 69.527}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.411, "count": 59, "max_ms": 78.008, "p95_ms": 77.575}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.803, "count": 59, "max_ms": 107.861, "p95_ms": 107.052}, "read_interval": {"avg_ms": 440.185, "count": 59, "max_ms": 19192.261, "p95_ms": 121.248}, "triton_wait": {"avg_ms": 73.423, "count": 59, "max_ms": 78.02, "p95_ms": 77.586}, "visualize": {"avg_ms": 3.365, "count": 59, "max_ms": 4.241, "p95_ms": 3.518}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11792}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10458.961, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55672c0255c0] error while decoding MB 76 29, bytestream -17 [in#0/rtsp @ 0x55672b690e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55672b690e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 181, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:32:37+09:00", "last_error_summary": "[h264 @ 0x55672c0255c0] error while decoding MB 76 29, bytestream -17 [in#0/rtsp @ 0x55672b690e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55672b690e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:23:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 571.652, "last_started_at": "2026-07-01T03:23:05+09:00", "pid": 75765, "process": {"exists": false, "pid": 75765}, "restart_count": 175, "restart_count_per_hour": 10, "rtsp_timeout_count": 286, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 176}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.768, "sample_count": 2482, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1415057, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:32:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-01T03:32:37+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T03:32:37+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T03:32:37.203474+09:00", "mtime_epoch": 1782844357.2034736, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93371}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:32:37+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1728.551, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 97, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.2, "pushed": 3.233, "read": 3.233}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.892, "count": 96, "max_ms": 23.621, "p95_ms": 20.886}, "gstreamer_push": {"avg_ms": 1.241, "count": 97, "max_ms": 1.378, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.658, "count": 96, "max_ms": 72.609, "p95_ms": 60.019}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.209, "count": 96, "max_ms": 70.495, "p95_ms": 68.779}, "yolo11_640_fire_11class": {"avg_ms": 63.366, "count": 96, "max_ms": 70.025, "p95_ms": 67.502}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.428, "count": 96, "max_ms": 79.851, "p95_ms": 78.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.858, "count": 96, "max_ms": 108.073, "p95_ms": 106.698}, "read_interval": {"avg_ms": 309.679, "count": 97, "max_ms": 10165.099, "p95_ms": 123.759}, "triton_wait": {"avg_ms": 72.447, "count": 96, "max_ms": 79.864, "p95_ms": 78.416}, "visualize": {"avg_ms": 3.451, "count": 96, "max_ms": 4.052, "p95_ms": 3.773}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11792}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10458.703, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:32:37+09:00", "last_error_summary": "[h264 @ 0x55672c0255c0] error while decoding MB 76 29, bytestream -17 [in#0/rtsp @ 0x55672b690e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55672b690e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 181, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:32:37+09:00", "last_error_summary": "[h264 @ 0x55672c0255c0] error while decoding MB 76 29, bytestream -17 [in#0/rtsp @ 0x55672b690e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55672b690e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:32:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 571.68, "last_started_at": "2026-07-01T03:23:05+09:00", "pid": 75765, "process": {"exists": false, "pid": 75765}, "restart_count": 175, "restart_count_per_hour": 10, "rtsp_timeout_count": 286, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 176}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.768, "sample_count": 2483, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1415058, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:32:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T03:32:37+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-01T03:32:37+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T03:32:37.203474+09:00", "mtime_epoch": 1782844357.2034736, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93371}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:32:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1728.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.243, "pushed": 3.243, "read": 3.21}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.951, "count": 97, "max_ms": 24.65, "p95_ms": 21.577}, "gstreamer_push": {"avg_ms": 1.242, "count": 97, "max_ms": 1.378, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.552, "count": 97, "max_ms": 72.609, "p95_ms": 60.019}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.02, "count": 97, "max_ms": 70.495, "p95_ms": 68.779}, "yolo11_640_fire_11class": {"avg_ms": 63.186, "count": 97, "max_ms": 70.025, "p95_ms": 67.502}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.165, "count": 97, "max_ms": 79.851, "p95_ms": 78.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.702, "count": 97, "max_ms": 108.073, "p95_ms": 106.698}, "read_interval": {"avg_ms": 311.668, "count": 96, "max_ms": 10165.099, "p95_ms": 123.759}, "triton_wait": {"avg_ms": 72.184, "count": 97, "max_ms": 79.864, "p95_ms": 78.416}, "visualize": {"avg_ms": 3.498, "count": 97, "max_ms": 8.025, "p95_ms": 3.782}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11792}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10458.52, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 181, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:32:37+09:00", "last_error_summary": "[h264 @ 0x55672c0255c0] error while decoding MB 76 29, bytestream -17 [in#0/rtsp @ 0x55672b690e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55672b690e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:32:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 571.688, "last_started_at": "2026-07-01T03:23:05+09:00", "pid": 75765, "process": {"exists": false, "pid": 75765}, "restart_count": 175, "restart_count_per_hour": 10, "rtsp_timeout_count": 286, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 176}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.768, "sample_count": 2483, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1415058, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:32:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T03:32:37+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-01T03:32:37+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-01T03:32:37.203474+09:00", "mtime_epoch": 1782844357.2034736, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93371}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:32:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1728.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.242, "pushed": 3.242, "read": 3.209}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.951, "count": 97, "max_ms": 24.65, "p95_ms": 21.577}, "gstreamer_push": {"avg_ms": 1.242, "count": 97, "max_ms": 1.378, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.552, "count": 97, "max_ms": 72.609, "p95_ms": 60.019}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.02, "count": 97, "max_ms": 70.495, "p95_ms": 68.779}, "yolo11_640_fire_11class": {"avg_ms": 63.186, "count": 97, "max_ms": 70.025, "p95_ms": 67.502}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.165, "count": 97, "max_ms": 79.851, "p95_ms": 78.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.702, "count": 97, "max_ms": 108.073, "p95_ms": 106.698}, "read_interval": {"avg_ms": 311.668, "count": 96, "max_ms": 10165.099, "p95_ms": 123.759}, "triton_wait": {"avg_ms": 72.184, "count": 97, "max_ms": 79.864, "p95_ms": 78.416}, "visualize": {"avg_ms": 3.498, "count": 97, "max_ms": 8.025, "p95_ms": 3.782}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11792}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10525.77, "limit_mb": 8796093022207.996}, "detail": {"pid": 76024, "restart_count": 176, "start_count": 177}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 181, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:32:37+09:00", "last_error_summary": "[h264 @ 0x55672c0255c0] error while decoding MB 76 29, bytestream -17 [in#0/rtsp @ 0x55672b690e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55672b690e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:32:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:32:39+09:00", "pid": 76024, "process": {"exists": true, "fd_count": 5, "hwm_mb": 99.918, "name": "ffmpeg", "pid": 76024, "rss_mb": 99.918, "state": "S (sleeping)", "threads": 19}, "restart_count": 176, "restart_count_per_hour": 11, "rtsp_timeout_count": 286, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 177}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.808, "sample_count": 2442, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1415058, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:32:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.701, "last_frame_pushed_at": "2026-07-01T03:32:37+09:00", "last_frame_read_age_sec": 4.788, "last_frame_read_at": "2026-07-01T03:32:37+09:00", "mjpeg": {"age_sec": 4.685, "exists": true, "mtime": "2026-07-01T03:32:37.303475+09:00", "mtime_epoch": 1782844357.3034754, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93015}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:32:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1728.551, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.935, "pushed": 1.935, "read": 1.902}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.038, "count": 58, "max_ms": 24.65, "p95_ms": 21.577}, "gstreamer_push": {"avg_ms": 1.235, "count": 58, "max_ms": 1.378, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.02, "count": 58, "max_ms": 60.344, "p95_ms": 54.05}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.44, "count": 58, "max_ms": 69.965, "p95_ms": 68.636}, "yolo11_640_fire_11class": {"avg_ms": 61.585, "count": 58, "max_ms": 67.508, "p95_ms": 67.488}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.791, "count": 58, "max_ms": 79.851, "p95_ms": 78.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.067, "count": 58, "max_ms": 106.738, "p95_ms": 105.768}, "read_interval": {"avg_ms": 443.733, "count": 57, "max_ms": 10165.099, "p95_ms": 124.316}, "triton_wait": {"avg_ms": 69.815, "count": 58, "max_ms": 79.864, "p95_ms": 78.416}, "visualize": {"avg_ms": 3.512, "count": 58, "max_ms": 8.025, "p95_ms": 3.616}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10648}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10501.391, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 4, bytestream 3300 [h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 7, bytestream 4292 [h264 @ 0x55da49953f40] error while decoding MB 32 42, bytestream -11 [h264 @ 0x55da498a0300] cabac decode of qscale diff failed at 3 15 [h264 @ 0x55da498a0300] error while decoding MB 3 15, bytestream 6124", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 371, "h264_decode_error_count": 1223, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:33:49+09:00", "last_error_summary": "[h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 4, bytestream 3300 [h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 7, bytestream 4292 [h264 @ 0x55da49953f40] error while decoding MB 32 42, bytestream -11 [h264 @ 0x55da498a0300] cabac decode of qscale diff failed at 3 15 [h264 @ 0x55da498a0300] error while decoding MB 3 15, bytestream 6124", "last_exit_at": "2026-07-01T03:28:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.614, "last_started_at": "2026-07-01T03:28:52+09:00", "pid": 75917, "process": {"exists": false, "pid": 75917}, "restart_count": 404, "restart_count_per_hour": 7, "rtsp_timeout_count": 360, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 405}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.006, "sample_count": 2483, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1277861, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:33:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-01T03:33:49+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-01T03:33:49+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-01T03:33:49.552680+09:00", "mtime_epoch": 1782844429.5526805, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120036}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:33:49+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4974.117, "name": "celery", "pid": 267, "rss_mb": 4974.117, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 93, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.105, "pushed": 3.105, "read": 3.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.389, "count": 93, "max_ms": 23.329, "p95_ms": 22.613}, "gstreamer_push": {"avg_ms": 1.197, "count": 93, "max_ms": 1.847, "p95_ms": 1.27}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.221, "count": 93, "max_ms": 44.886, "p95_ms": 41.431}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.141, "count": 93, "max_ms": 76.646, "p95_ms": 72.841}, "yolo11_640_fire_11class": {"avg_ms": 65.554, "count": 93, "max_ms": 74.524, "p95_ms": 73.577}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.277, "count": 93, "max_ms": 83.432, "p95_ms": 78.564}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.694, "count": 93, "max_ms": 112.13, "p95_ms": 108.356}, "read_interval": {"avg_ms": 322.537, "count": 93, "max_ms": 19100.524, "p95_ms": 125.478}, "triton_wait": {"avg_ms": 74.327, "count": 93, "max_ms": 83.446, "p95_ms": 78.575}, "visualize": {"avg_ms": 3.358, "count": 93, "max_ms": 3.848, "p95_ms": 3.702}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10648}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10500.938, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:33:49+09:00", "last_error_summary": "[h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 4, bytestream 3300 [h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 7, bytestream 4292 [h264 @ 0x55da49953f40] error while decoding MB 32 42, bytestream -11 [h264 @ 0x55da498a0300] cabac decode of qscale diff failed at 3 15 [h264 @ 0x55da498a0300] error while decoding MB 3 15, bytestream 6124", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 371, "h264_decode_error_count": 1223, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:33:49+09:00", "last_error_summary": "[h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 4, bytestream 3300 [h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 7, bytestream 4292 [h264 @ 0x55da49953f40] error while decoding MB 32 42, bytestream -11 [h264 @ 0x55da498a0300] cabac decode of qscale diff failed at 3 15 [h264 @ 0x55da498a0300] error while decoding MB 3 15, bytestream 6124", "last_exit_at": "2026-07-01T03:33:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.666, "last_started_at": "2026-07-01T03:28:52+09:00", "pid": 75917, "process": {"exists": false, "pid": 75917}, "restart_count": 404, "restart_count_per_hour": 7, "rtsp_timeout_count": 360, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 405}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.006, "sample_count": 2484, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1277862, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:33:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T03:33:49+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-01T03:33:49+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-01T03:33:49.552680+09:00", "mtime_epoch": 1782844429.5526805, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120036}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:33:49+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4974.117, "name": "celery", "pid": 267, "rss_mb": 4974.117, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.1, "pushed": 3.133, "read": 3.1}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.381, "count": 93, "max_ms": 23.329, "p95_ms": 22.613}, "gstreamer_push": {"avg_ms": 1.197, "count": 94, "max_ms": 1.847, "p95_ms": 1.27}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.335, "count": 93, "max_ms": 44.886, "p95_ms": 41.566}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.131, "count": 93, "max_ms": 76.646, "p95_ms": 72.841}, "yolo11_640_fire_11class": {"avg_ms": 65.552, "count": 93, "max_ms": 74.524, "p95_ms": 73.577}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.259, "count": 93, "max_ms": 83.432, "p95_ms": 78.564}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.672, "count": 93, "max_ms": 112.13, "p95_ms": 108.356}, "read_interval": {"avg_ms": 322.537, "count": 93, "max_ms": 19100.524, "p95_ms": 125.478}, "triton_wait": {"avg_ms": 74.309, "count": 93, "max_ms": 83.446, "p95_ms": 78.575}, "visualize": {"avg_ms": 3.359, "count": 93, "max_ms": 3.848, "p95_ms": 3.702}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10648}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10501.07, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 371, "h264_decode_error_count": 1223, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:33:49+09:00", "last_error_summary": "[h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 4, bytestream 3300 [h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 7, bytestream 4292 [h264 @ 0x55da49953f40] error while decoding MB 32 42, bytestream -11 [h264 @ 0x55da498a0300] cabac decode of qscale diff failed at 3 15 [h264 @ 0x55da498a0300] error while decoding MB 3 15, bytestream 6124", "last_exit_at": "2026-07-01T03:33:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.672, "last_started_at": "2026-07-01T03:28:52+09:00", "pid": 75917, "process": {"exists": false, "pid": 75917}, "restart_count": 404, "restart_count_per_hour": 7, "rtsp_timeout_count": 360, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 405}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.006, "sample_count": 2484, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1277862, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:33:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T03:33:49+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-01T03:33:49+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-01T03:33:49.552680+09:00", "mtime_epoch": 1782844429.5526805, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120036}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:33:49+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4974.117, "name": "celery", "pid": 267, "rss_mb": 4974.117, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 93, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.101, "pushed": 3.101, "read": 3.101}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.381, "count": 93, "max_ms": 23.329, "p95_ms": 22.613}, "gstreamer_push": {"avg_ms": 1.197, "count": 93, "max_ms": 1.847, "p95_ms": 1.27}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.335, "count": 93, "max_ms": 44.886, "p95_ms": 41.566}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.131, "count": 93, "max_ms": 76.646, "p95_ms": 72.841}, "yolo11_640_fire_11class": {"avg_ms": 65.552, "count": 93, "max_ms": 74.524, "p95_ms": 73.577}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.259, "count": 93, "max_ms": 83.432, "p95_ms": 78.564}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.672, "count": 93, "max_ms": 112.13, "p95_ms": 108.356}, "read_interval": {"avg_ms": 322.537, "count": 93, "max_ms": 19100.524, "p95_ms": 125.478}, "triton_wait": {"avg_ms": 74.309, "count": 93, "max_ms": 83.446, "p95_ms": 78.575}, "visualize": {"avg_ms": 3.359, "count": 93, "max_ms": 3.848, "p95_ms": 3.702}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10648}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10608.289, "limit_mb": 8796093022207.996}, "detail": {"pid": 76176, "restart_count": 405, "start_count": 406}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 371, "h264_decode_error_count": 1223, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:33:49+09:00", "last_error_summary": "[h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 4, bytestream 3300 [h264 @ 0x55da49874dc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55da49874dc0] error while decoding MB 0 7, bytestream 4292 [h264 @ 0x55da49953f40] error while decoding MB 32 42, bytestream -11 [h264 @ 0x55da498a0300] cabac decode of qscale diff failed at 3 15 [h264 @ 0x55da498a0300] error while decoding MB 3 15, bytestream 6124", "last_exit_at": "2026-07-01T03:33:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:33:52+09:00", "pid": 76176, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.648, "name": "ffmpeg", "pid": 76176, "rss_mb": 138.648, "state": "S (sleeping)", "threads": 106}, "restart_count": 405, "restart_count_per_hour": 8, "rtsp_timeout_count": 360, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 406}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.006, "sample_count": 2472, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1277862, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:33:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.656, "last_frame_pushed_at": "2026-07-01T03:33:49+09:00", "last_frame_read_age_sec": 4.759, "last_frame_read_at": "2026-07-01T03:33:49+09:00", "mjpeg": {"age_sec": 4.643, "exists": true, "mtime": "2026-07-01T03:33:49.668683+09:00", "mtime_epoch": 1782844429.6686826, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116703}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:33:54+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4974.117, "name": "celery", "pid": 267, "rss_mb": 4974.117, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 54, "pushed_frames": 54, "read_frames": 53, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.803, "pushed": 1.803, "read": 1.77}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.163, "count": 54, "max_ms": 23.329, "p95_ms": 23.013}, "gstreamer_push": {"avg_ms": 1.212, "count": 54, "max_ms": 1.847, "p95_ms": 1.285}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.965, "count": 54, "max_ms": 44.886, "p95_ms": 41.431}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.552, "count": 54, "max_ms": 76.646, "p95_ms": 74.423}, "yolo11_640_fire_11class": {"avg_ms": 67.472, "count": 54, "max_ms": 74.524, "p95_ms": 73.822}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.286, "count": 54, "max_ms": 83.432, "p95_ms": 78.839}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.463, "count": 54, "max_ms": 112.13, "p95_ms": 108.517}, "read_interval": {"avg_ms": 477.202, "count": 53, "max_ms": 19100.524, "p95_ms": 125.729}, "triton_wait": {"avg_ms": 75.363, "count": 54, "max_ms": 83.446, "p95_ms": 78.852}, "visualize": {"avg_ms": 3.325, "count": 54, "max_ms": 3.848, "p95_ms": 3.719}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11819}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10752.754, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 181, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:32:37+09:00", "last_error_summary": "[h264 @ 0x55672c0255c0] error while decoding MB 76 29, bytestream -17 [in#0/rtsp @ 0x55672b690e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55672b690e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:32:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 379.17, "last_started_at": "2026-07-01T03:32:39+09:00", "pid": 76024, "process": {"exists": true, "fd_count": 5, "hwm_mb": 326.234, "name": "ffmpeg", "pid": 76024, "rss_mb": 326.234, "state": "S (sleeping)", "threads": 106}, "restart_count": 176, "restart_count_per_hour": 9, "rtsp_timeout_count": 286, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 177}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.531, "sample_count": 2488, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1418319, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:38:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.057, "last_frame_pushed_at": "2026-07-01T03:38:29+09:00", "last_frame_read_age_sec": 30.162, "last_frame_read_at": "2026-07-01T03:38:28+09:00", "mjpeg": {"age_sec": 30.04, "exists": true, "mtime": "2026-07-01T03:38:29.117434+09:00", "mtime_epoch": 1782844709.1174338, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98611}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:38:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1730.098, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11819}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10744.852, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 181, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:32:37+09:00", "last_error_summary": "[h264 @ 0x55672c0255c0] error while decoding MB 76 29, bytestream -17 [in#0/rtsp @ 0x55672b690e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55672b690e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:32:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 379.183, "last_started_at": "2026-07-01T03:32:39+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 176, "restart_count_per_hour": 9, "rtsp_timeout_count": 286, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 177}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.531, "sample_count": 2488, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1418319, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:38:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.07, "last_frame_pushed_at": "2026-07-01T03:38:29+09:00", "last_frame_read_age_sec": 30.175, "last_frame_read_at": "2026-07-01T03:38:28+09:00", "mjpeg": {"age_sec": 30.054, "exists": true, "mtime": "2026-07-01T03:38:29.117434+09:00", "mtime_epoch": 1782844709.1174338, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98611}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:38:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1730.098, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11819}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10454.348, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fec525bf00] error while decoding MB 107 31, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 177, "h264_decode_error_count": 182, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:38:59+09:00", "last_error_summary": "[h264 @ 0x55fec525bf00] error while decoding MB 107 31, bytestream -5", "last_exit_at": "2026-07-01T03:32:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 379.233, "last_started_at": "2026-07-01T03:32:39+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 176, "restart_count_per_hour": 9, "rtsp_timeout_count": 286, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 177}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.531, "sample_count": 2488, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1418319, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:38:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.12, "last_frame_pushed_at": "2026-07-01T03:38:29+09:00", "last_frame_read_age_sec": 30.225, "last_frame_read_at": "2026-07-01T03:38:28+09:00", "mjpeg": {"age_sec": 30.104, "exists": true, "mtime": "2026-07-01T03:38:29.117434+09:00", "mtime_epoch": 1782844709.1174338, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98611}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:38:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1730.098, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11819}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10571.691, "limit_mb": 8796093022207.996}, "detail": {"pid": 76283, "restart_count": 177, "start_count": 178}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 177, "h264_decode_error_count": 182, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:38:59+09:00", "last_error_summary": "[h264 @ 0x55fec525bf00] error while decoding MB 107 31, bytestream -5", "last_exit_at": "2026-07-01T03:32:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:39:02+09:00", "pid": 76283, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.98, "name": "ffmpeg", "pid": 76283, "rss_mb": 147.98, "state": "R (running)", "threads": 49}, "restart_count": 177, "restart_count_per_hour": 10, "rtsp_timeout_count": 286, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 178}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.39, "sample_count": 2441, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1418319, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:38:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 34.967, "last_frame_pushed_at": "2026-07-01T03:38:29+09:00", "last_frame_read_age_sec": 35.071, "last_frame_read_at": "2026-07-01T03:38:28+09:00", "mjpeg": {"age_sec": 34.95, "exists": true, "mtime": "2026-07-01T03:38:29.117434+09:00", "mtime_epoch": 1782844709.1174338, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98611}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:39:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1730.098, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11833}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10479.074, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x559b4f020200] missing picture in access unit with size 18 [h264 @ 0x559b4f8fcf80] no frame! [in#0/rtsp @ 0x559b4f01be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b4f01be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x559b4f020840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 178, "h264_decode_error_count": 183, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:42:31+09:00", "last_error_summary": "[NULL @ 0x559b4f020200] missing picture in access unit with size 18 [h264 @ 0x559b4f8fcf80] no frame! [in#0/rtsp @ 0x559b4f01be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b4f01be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x559b4f020840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T03:32:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.123, "last_started_at": "2026-07-01T03:39:02+09:00", "pid": 76283, "process": {"exists": false, "pid": 76283}, "restart_count": 177, "restart_count_per_hour": 9, "rtsp_timeout_count": 288, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 178}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.236, "sample_count": 2176, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1419976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:42:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T03:42:31+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T03:42:31+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-01T03:42:31.129627+09:00", "mtime_epoch": 1782844951.1296275, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 109109}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:42:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1730.871, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 28, "pushed_frames": 28, "read_frames": 28, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.935, "pushed": 0.935, "read": 0.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.753, "count": 28, "max_ms": 25.279, "p95_ms": 25.065}, "gstreamer_push": {"avg_ms": 1.219, "count": 28, "max_ms": 1.471, "p95_ms": 1.438}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.031, "count": 28, "max_ms": 43.224, "p95_ms": 42.751}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.416, "count": 28, "max_ms": 55.426, "p95_ms": 49.424}, "yolo11_640_fire_11class": {"avg_ms": 46.354, "count": 28, "max_ms": 55.415, "p95_ms": 48.768}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.727, "count": 28, "max_ms": 52.936, "p95_ms": 50.804}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.147, "count": 28, "max_ms": 84.041, "p95_ms": 82.224}, "read_interval": {"avg_ms": 1069.712, "count": 28, "max_ms": 15991.211, "p95_ms": 11603.079}, "triton_wait": {"avg_ms": 48.232, "count": 28, "max_ms": 55.437, "p95_ms": 50.817}, "visualize": {"avg_ms": 3.499, "count": 28, "max_ms": 4.439, "p95_ms": 4.314}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11833}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10478.703, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:42:31+09:00", "last_error_summary": "[NULL @ 0x559b4f020200] missing picture in access unit with size 18 [h264 @ 0x559b4f8fcf80] no frame! [in#0/rtsp @ 0x559b4f01be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b4f01be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x559b4f020840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 178, "h264_decode_error_count": 183, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:42:31+09:00", "last_error_summary": "[NULL @ 0x559b4f020200] missing picture in access unit with size 18 [h264 @ 0x559b4f8fcf80] no frame! [in#0/rtsp @ 0x559b4f01be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b4f01be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x559b4f020840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T03:42:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.146, "last_started_at": "2026-07-01T03:39:02+09:00", "pid": 76283, "process": {"exists": false, "pid": 76283}, "restart_count": 177, "restart_count_per_hour": 9, "rtsp_timeout_count": 288, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 178}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.236, "sample_count": 2177, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1419977, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:42:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T03:42:31+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-01T03:42:31+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-01T03:42:31.129627+09:00", "mtime_epoch": 1782844951.1296275, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 109109}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:42:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1730.871, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 29, "pushed_frames": 29, "read_frames": 28, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.968, "pushed": 0.968, "read": 0.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.735, "count": 29, "max_ms": 25.279, "p95_ms": 25.065}, "gstreamer_push": {"avg_ms": 1.222, "count": 29, "max_ms": 1.471, "p95_ms": 1.438}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.044, "count": 29, "max_ms": 43.224, "p95_ms": 42.751}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.41, "count": 29, "max_ms": 55.426, "p95_ms": 49.424}, "yolo11_640_fire_11class": {"avg_ms": 46.361, "count": 29, "max_ms": 55.415, "p95_ms": 48.768}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.727, "count": 29, "max_ms": 52.936, "p95_ms": 50.804}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.205, "count": 29, "max_ms": 84.041, "p95_ms": 82.224}, "read_interval": {"avg_ms": 1069.712, "count": 28, "max_ms": 15991.211, "p95_ms": 11603.079}, "triton_wait": {"avg_ms": 48.215, "count": 29, "max_ms": 55.437, "p95_ms": 50.817}, "visualize": {"avg_ms": 3.584, "count": 29, "max_ms": 5.959, "p95_ms": 4.439}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11833}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10478.695, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 178, "h264_decode_error_count": 183, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:42:31+09:00", "last_error_summary": "[NULL @ 0x559b4f020200] missing picture in access unit with size 18 [h264 @ 0x559b4f8fcf80] no frame! [in#0/rtsp @ 0x559b4f01be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b4f01be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x559b4f020840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T03:42:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.153, "last_started_at": "2026-07-01T03:39:02+09:00", "pid": 76283, "process": {"exists": false, "pid": 76283}, "restart_count": 177, "restart_count_per_hour": 9, "rtsp_timeout_count": 288, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 178}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.236, "sample_count": 2177, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1419977, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:42:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T03:42:31+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-01T03:42:31+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-01T03:42:31.129627+09:00", "mtime_epoch": 1782844951.1296275, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 109109}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:42:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1730.871, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 29, "pushed_frames": 29, "read_frames": 28, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.968, "pushed": 0.968, "read": 0.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.735, "count": 29, "max_ms": 25.279, "p95_ms": 25.065}, "gstreamer_push": {"avg_ms": 1.222, "count": 29, "max_ms": 1.471, "p95_ms": 1.438}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.044, "count": 29, "max_ms": 43.224, "p95_ms": 42.751}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.41, "count": 29, "max_ms": 55.426, "p95_ms": 49.424}, "yolo11_640_fire_11class": {"avg_ms": 46.361, "count": 29, "max_ms": 55.415, "p95_ms": 48.768}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.727, "count": 29, "max_ms": 52.936, "p95_ms": 50.804}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.205, "count": 29, "max_ms": 84.041, "p95_ms": 82.224}, "read_interval": {"avg_ms": 1069.712, "count": 28, "max_ms": 15991.211, "p95_ms": 11603.079}, "triton_wait": {"avg_ms": 48.215, "count": 29, "max_ms": 55.437, "p95_ms": 50.817}, "visualize": {"avg_ms": 3.584, "count": 29, "max_ms": 5.959, "p95_ms": 4.439}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11833}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10534.648, "limit_mb": 8796093022207.996}, "detail": {"pid": 76435, "restart_count": 178, "start_count": 179}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 178, "h264_decode_error_count": 183, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:42:31+09:00", "last_error_summary": "[NULL @ 0x559b4f020200] missing picture in access unit with size 18 [h264 @ 0x559b4f8fcf80] no frame! [in#0/rtsp @ 0x559b4f01be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b4f01be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x559b4f020840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T03:42:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:42:33+09:00", "pid": 76435, "process": {"exists": true, "fd_count": 5, "hwm_mb": 90.012, "name": "ffmpeg", "pid": 76435, "rss_mb": 90.012, "state": "S (sleeping)", "threads": 19}, "restart_count": 178, "restart_count_per_hour": 10, "rtsp_timeout_count": 288, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 179}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.207, "sample_count": 2136, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1419977, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:42:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.778, "last_frame_pushed_at": "2026-07-01T03:42:31+09:00", "last_frame_read_age_sec": 4.859, "last_frame_read_at": "2026-07-01T03:42:31+09:00", "mjpeg": {"age_sec": 4.764, "exists": true, "mtime": "2026-07-01T03:42:31.221629+09:00", "mtime_epoch": 1782844951.221629, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 109953}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:42:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1730.871, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.596, "pushed": 0.596, "read": 0.596}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.383, "count": 13, "max_ms": 20.278, "p95_ms": 20.278}, "gstreamer_push": {"avg_ms": 1.232, "count": 13, "max_ms": 1.289, "p95_ms": 1.289}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.373, "count": 13, "max_ms": 43.224, "p95_ms": 43.224}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.562, "count": 13, "max_ms": 55.426, "p95_ms": 55.426}, "yolo11_640_fire_11class": {"avg_ms": 46.679, "count": 13, "max_ms": 55.415, "p95_ms": 55.415}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.625, "count": 13, "max_ms": 52.936, "p95_ms": 52.936}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.706, "count": 13, "max_ms": 84.041, "p95_ms": 84.041}, "read_interval": {"avg_ms": 2195.689, "count": 13, "max_ms": 15991.211, "p95_ms": 15991.211}, "triton_wait": {"avg_ms": 48.044, "count": 13, "max_ms": 55.437, "p95_ms": 55.437}, "visualize": {"avg_ms": 3.63, "count": 13, "max_ms": 5.959, "p95_ms": 5.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.8, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10504.164, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a80355f1c0] error while decoding MB 78 33, bytestream -35 [h264 @ 0x55a803490400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a803490400] error while decoding MB 0 5, bytestream 3807 [h264 @ 0x55a80355f1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a80355f1c0] error while decoding MB 0 4, bytestream 4487 [h264 @ 0x55a802b55fc0] error while decoding MB 58 21, bytestream -13 [rtsp @ 0x55a802ae6f00] RTP: PT=60: bad cseq 5254 expected=16bd [h", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 372, "h264_decode_error_count": 1228, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:42:42+09:00", "last_error_summary": "[h264 @ 0x55a80355f1c0] error while decoding MB 78 33, bytestream -35 [h264 @ 0x55a803490400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a803490400] error while decoding MB 0 5, bytestream 3807 [h264 @ 0x55a80355f1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a80355f1c0] error while decoding MB 0 4, bytestream 4487 [h264 @ 0x55a802b55fc0] error while decoding MB 58 21, bytestream -13 [rtsp @ 0x55a802ae6f00] RTP: PT=60: bad cseq 5254 expected=16bd [h", "last_exit_at": "2026-07-01T03:33:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 529.881, "last_started_at": "2026-07-01T03:33:52+09:00", "pid": 76176, "process": {"exists": false, "pid": 76176}, "restart_count": 405, "restart_count_per_hour": 7, "rtsp_timeout_count": 362, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 406}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.717, "sample_count": 2651, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1282627, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:42:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-01T03:42:42+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T03:42:42+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-01T03:42:42.125819+09:00", "mtime_epoch": 1782844962.1258192, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 132825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:42:42+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4976.438, "name": "celery", "pid": 267, "rss_mb": 4976.438, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 112, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.736, "pushed": 3.736, "read": 3.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.952, "count": 112, "max_ms": 24.798, "p95_ms": 22.126}, "gstreamer_push": {"avg_ms": 1.214, "count": 112, "max_ms": 1.597, "p95_ms": 1.461}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.821, "count": 112, "max_ms": 35.968, "p95_ms": 33.755}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.525, "count": 112, "max_ms": 49.729, "p95_ms": 47.697}, "yolo11_640_fire_11class": {"avg_ms": 43.736, "count": 112, "max_ms": 47.171, "p95_ms": 46.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.553, "count": 112, "max_ms": 46.384, "p95_ms": 45.659}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.155, "count": 112, "max_ms": 79.717, "p95_ms": 77.483}, "read_interval": {"avg_ms": 267.724, "count": 112, "max_ms": 10123.838, "p95_ms": 93.476}, "triton_wait": {"avg_ms": 45.217, "count": 112, "max_ms": 49.743, "p95_ms": 47.713}, "visualize": {"avg_ms": 3.413, "count": 112, "max_ms": 5.014, "p95_ms": 3.753}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10504.035, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:42:42+09:00", "last_error_summary": "[h264 @ 0x55a80355f1c0] error while decoding MB 78 33, bytestream -35 [h264 @ 0x55a803490400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a803490400] error while decoding MB 0 5, bytestream 3807 [h264 @ 0x55a80355f1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a80355f1c0] error while decoding MB 0 4, bytestream 4487 [h264 @ 0x55a802b55fc0] error while decoding MB 58 21, bytestream -13 [rtsp @ 0x55a802ae6f00] RTP: PT=60: bad cseq 5254 expected=16bd [h", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 372, "h264_decode_error_count": 1228, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:42:42+09:00", "last_error_summary": "[h264 @ 0x55a80355f1c0] error while decoding MB 78 33, bytestream -35 [h264 @ 0x55a803490400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a803490400] error while decoding MB 0 5, bytestream 3807 [h264 @ 0x55a80355f1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a80355f1c0] error while decoding MB 0 4, bytestream 4487 [h264 @ 0x55a802b55fc0] error while decoding MB 58 21, bytestream -13 [rtsp @ 0x55a802ae6f00] RTP: PT=60: bad cseq 5254 expected=16bd [h", "last_exit_at": "2026-07-01T03:42:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 529.908, "last_started_at": "2026-07-01T03:33:52+09:00", "pid": 76176, "process": {"exists": false, "pid": 76176}, "restart_count": 405, "restart_count_per_hour": 7, "rtsp_timeout_count": 362, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 406}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.698, "sample_count": 2651, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1282628, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:42:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T03:42:42+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-01T03:42:42+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-01T03:42:42.125819+09:00", "mtime_epoch": 1782844962.1258192, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 132825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:42:42+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4976.438, "name": "celery", "pid": 267, "rss_mb": 4976.438, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 112, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.735, "pushed": 3.735, "read": 3.735}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.996, "count": 112, "max_ms": 25.609, "p95_ms": 22.693}, "gstreamer_push": {"avg_ms": 1.215, "count": 112, "max_ms": 1.597, "p95_ms": 1.461}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.826, "count": 112, "max_ms": 35.968, "p95_ms": 33.755}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.508, "count": 112, "max_ms": 49.729, "p95_ms": 47.697}, "yolo11_640_fire_11class": {"avg_ms": 43.745, "count": 112, "max_ms": 47.171, "p95_ms": 46.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.583, "count": 112, "max_ms": 46.488, "p95_ms": 45.674}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.225, "count": 112, "max_ms": 83.632, "p95_ms": 77.534}, "read_interval": {"avg_ms": 267.724, "count": 112, "max_ms": 10123.838, "p95_ms": 93.476}, "triton_wait": {"avg_ms": 45.208, "count": 112, "max_ms": 49.743, "p95_ms": 47.713}, "visualize": {"avg_ms": 3.44, "count": 112, "max_ms": 6.451, "p95_ms": 3.833}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10503.898, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 372, "h264_decode_error_count": 1228, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:42:42+09:00", "last_error_summary": "[h264 @ 0x55a80355f1c0] error while decoding MB 78 33, bytestream -35 [h264 @ 0x55a803490400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a803490400] error while decoding MB 0 5, bytestream 3807 [h264 @ 0x55a80355f1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a80355f1c0] error while decoding MB 0 4, bytestream 4487 [h264 @ 0x55a802b55fc0] error while decoding MB 58 21, bytestream -13 [rtsp @ 0x55a802ae6f00] RTP: PT=60: bad cseq 5254 expected=16bd [h", "last_exit_at": "2026-07-01T03:42:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 529.916, "last_started_at": "2026-07-01T03:33:52+09:00", "pid": 76176, "process": {"exists": false, "pid": 76176}, "restart_count": 405, "restart_count_per_hour": 7, "rtsp_timeout_count": 362, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 406}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.698, "sample_count": 2651, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1282628, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:42:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T03:42:42+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-01T03:42:42+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-01T03:42:42.125819+09:00", "mtime_epoch": 1782844962.1258192, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 132825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:42:42+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4976.438, "name": "celery", "pid": 267, "rss_mb": 4976.438, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 112, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.734, "pushed": 3.734, "read": 3.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.996, "count": 112, "max_ms": 25.609, "p95_ms": 22.693}, "gstreamer_push": {"avg_ms": 1.215, "count": 112, "max_ms": 1.597, "p95_ms": 1.461}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.826, "count": 112, "max_ms": 35.968, "p95_ms": 33.755}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.508, "count": 112, "max_ms": 49.729, "p95_ms": 47.697}, "yolo11_640_fire_11class": {"avg_ms": 43.745, "count": 112, "max_ms": 47.171, "p95_ms": 46.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.583, "count": 112, "max_ms": 46.488, "p95_ms": 45.674}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.225, "count": 112, "max_ms": 83.632, "p95_ms": 77.534}, "read_interval": {"avg_ms": 267.724, "count": 112, "max_ms": 10123.838, "p95_ms": 93.476}, "triton_wait": {"avg_ms": 45.208, "count": 112, "max_ms": 49.743, "p95_ms": 47.713}, "visualize": {"avg_ms": 3.44, "count": 112, "max_ms": 6.451, "p95_ms": 3.833}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10612.395, "limit_mb": 8796093022207.996}, "detail": {"pid": 76587, "restart_count": 406, "start_count": 407}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 372, "h264_decode_error_count": 1228, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:42:42+09:00", "last_error_summary": "[h264 @ 0x55a80355f1c0] error while decoding MB 78 33, bytestream -35 [h264 @ 0x55a803490400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a803490400] error while decoding MB 0 5, bytestream 3807 [h264 @ 0x55a80355f1c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a80355f1c0] error while decoding MB 0 4, bytestream 4487 [h264 @ 0x55a802b55fc0] error while decoding MB 58 21, bytestream -13 [rtsp @ 0x55a802ae6f00] RTP: PT=60: bad cseq 5254 expected=16bd [h", "last_exit_at": "2026-07-01T03:42:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:42:44+09:00", "pid": 76587, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.711, "name": "ffmpeg", "pid": 76587, "rss_mb": 138.711, "state": "S (sleeping)", "threads": 106}, "restart_count": 406, "restart_count_per_hour": 8, "rtsp_timeout_count": 362, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 407}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.253, "sample_count": 2617, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1282628, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:42:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.328, "last_frame_pushed_at": "2026-07-01T03:42:42+09:00", "last_frame_read_age_sec": 4.414, "last_frame_read_at": "2026-07-01T03:42:42+09:00", "mjpeg": {"age_sec": 4.314, "exists": true, "mtime": "2026-07-01T03:42:42.225821+09:00", "mtime_epoch": 1782844962.2258208, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 134494}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:42:46+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4976.438, "name": "celery", "pid": 267, "rss_mb": 4976.438, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.167, "pushed": 2.167, "read": 2.133}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.944, "count": 64, "max_ms": 25.609, "p95_ms": 22.047}, "gstreamer_push": {"avg_ms": 1.221, "count": 65, "max_ms": 1.597, "p95_ms": 1.473}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.631, "count": 64, "max_ms": 33.847, "p95_ms": 32.559}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 43.701, "count": 64, "max_ms": 46.609, "p95_ms": 45.466}, "yolo11_640_fire_11class": {"avg_ms": 43.166, "count": 64, "max_ms": 46.479, "p95_ms": 44.144}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.522, "count": 64, "max_ms": 46.488, "p95_ms": 45.606}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.626, "count": 65, "max_ms": 83.632, "p95_ms": 76.847}, "read_interval": {"avg_ms": 400.916, "count": 64, "max_ms": 10123.838, "p95_ms": 92.953}, "triton_wait": {"avg_ms": 44.595, "count": 64, "max_ms": 46.62, "p95_ms": 45.704}, "visualize": {"avg_ms": 3.46, "count": 64, "max_ms": 6.451, "p95_ms": 4.096}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11847}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10471.297, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556c4e99f400] error while decoding MB 89 26, bytestream -17 [in#0/rtsp @ 0x556c4dbece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c4dbece00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 179, "h264_decode_error_count": 184, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:46:15+09:00", "last_error_summary": "[h264 @ 0x556c4e99f400] error while decoding MB 89 26, bytestream -17 [in#0/rtsp @ 0x556c4dbece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c4dbece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:42:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 221.443, "last_started_at": "2026-07-01T03:42:33+09:00", "pid": 76435, "process": {"exists": false, "pid": 76435}, "restart_count": 178, "restart_count_per_hour": 9, "rtsp_timeout_count": 290, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 179}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.885, "sample_count": 2246, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1421725, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:46:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-01T03:46:15+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T03:46:15+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-01T03:46:15.361553+09:00", "mtime_epoch": 1782845175.3615527, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105517}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:46:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1731.645, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 87, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.867, "pushed": 2.867, "read": 2.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.74, "count": 86, "max_ms": 22.26, "p95_ms": 21.645}, "gstreamer_push": {"avg_ms": 1.207, "count": 86, "max_ms": 1.452, "p95_ms": 1.281}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.184, "count": 86, "max_ms": 51.643, "p95_ms": 48.324}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.534, "count": 86, "max_ms": 55.337, "p95_ms": 52.313}, "yolo11_640_fire_11class": {"avg_ms": 46.911, "count": 86, "max_ms": 53.635, "p95_ms": 51.529}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.178, "count": 86, "max_ms": 55.349, "p95_ms": 52.579}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.988, "count": 86, "max_ms": 85.783, "p95_ms": 81.124}, "read_interval": {"avg_ms": 344.942, "count": 87, "max_ms": 13017.407, "p95_ms": 98.391}, "triton_wait": {"avg_ms": 48.22, "count": 86, "max_ms": 55.359, "p95_ms": 52.588}, "visualize": {"avg_ms": 3.42, "count": 86, "max_ms": 3.968, "p95_ms": 3.58}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11847}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10470.965, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:46:15+09:00", "last_error_summary": "[h264 @ 0x556c4e99f400] error while decoding MB 89 26, bytestream -17 [in#0/rtsp @ 0x556c4dbece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c4dbece00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 179, "h264_decode_error_count": 184, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:46:15+09:00", "last_error_summary": "[h264 @ 0x556c4e99f400] error while decoding MB 89 26, bytestream -17 [in#0/rtsp @ 0x556c4dbece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c4dbece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:46:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 221.47, "last_started_at": "2026-07-01T03:42:33+09:00", "pid": 76435, "process": {"exists": false, "pid": 76435}, "restart_count": 178, "restart_count_per_hour": 9, "rtsp_timeout_count": 290, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 179}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.885, "sample_count": 2247, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1421726, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:46:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T03:46:15+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-01T03:46:15+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-01T03:46:15.361553+09:00", "mtime_epoch": 1782845175.3615527, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105517}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:46:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1731.645, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.905, "pushed": 2.905, "read": 2.871}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.81, "count": 87, "max_ms": 26.8, "p95_ms": 21.73}, "gstreamer_push": {"avg_ms": 1.209, "count": 87, "max_ms": 1.452, "p95_ms": 1.29}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.183, "count": 87, "max_ms": 51.643, "p95_ms": 48.324}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.543, "count": 87, "max_ms": 55.337, "p95_ms": 52.313}, "yolo11_640_fire_11class": {"avg_ms": 46.926, "count": 87, "max_ms": 53.635, "p95_ms": 51.529}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.185, "count": 87, "max_ms": 55.349, "p95_ms": 52.579}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.09, "count": 87, "max_ms": 85.843, "p95_ms": 81.327}, "read_interval": {"avg_ms": 347.955, "count": 86, "max_ms": 13017.407, "p95_ms": 98.391}, "triton_wait": {"avg_ms": 48.227, "count": 87, "max_ms": 55.359, "p95_ms": 52.588}, "visualize": {"avg_ms": 3.441, "count": 87, "max_ms": 5.254, "p95_ms": 3.62}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11847}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10470.828, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 179, "h264_decode_error_count": 184, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:46:15+09:00", "last_error_summary": "[h264 @ 0x556c4e99f400] error while decoding MB 89 26, bytestream -17 [in#0/rtsp @ 0x556c4dbece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c4dbece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:46:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 221.478, "last_started_at": "2026-07-01T03:42:33+09:00", "pid": 76435, "process": {"exists": false, "pid": 76435}, "restart_count": 178, "restart_count_per_hour": 9, "rtsp_timeout_count": 290, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 179}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.885, "sample_count": 2247, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1421726, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:46:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-01T03:46:15+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-01T03:46:15+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-01T03:46:15.361553+09:00", "mtime_epoch": 1782845175.3615527, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105517}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:46:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1731.645, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.904, "pushed": 2.904, "read": 2.871}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.81, "count": 87, "max_ms": 26.8, "p95_ms": 21.73}, "gstreamer_push": {"avg_ms": 1.209, "count": 87, "max_ms": 1.452, "p95_ms": 1.29}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.183, "count": 87, "max_ms": 51.643, "p95_ms": 48.324}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.543, "count": 87, "max_ms": 55.337, "p95_ms": 52.313}, "yolo11_640_fire_11class": {"avg_ms": 46.926, "count": 87, "max_ms": 53.635, "p95_ms": 51.529}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.185, "count": 87, "max_ms": 55.349, "p95_ms": 52.579}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.09, "count": 87, "max_ms": 85.843, "p95_ms": 81.327}, "read_interval": {"avg_ms": 347.955, "count": 86, "max_ms": 13017.407, "p95_ms": 98.391}, "triton_wait": {"avg_ms": 48.227, "count": 87, "max_ms": 55.359, "p95_ms": 52.588}, "visualize": {"avg_ms": 3.441, "count": 87, "max_ms": 5.254, "p95_ms": 3.62}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11847}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10622.93, "limit_mb": 8796093022207.996}, "detail": {"pid": 76694, "restart_count": 179, "start_count": 180}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 179, "h264_decode_error_count": 184, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:46:15+09:00", "last_error_summary": "[h264 @ 0x556c4e99f400] error while decoding MB 89 26, bytestream -17 [in#0/rtsp @ 0x556c4dbece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556c4dbece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:46:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:46:18+09:00", "pid": 76694, "process": {"exists": true, "fd_count": 5, "hwm_mb": 177.305, "name": "ffmpeg", "pid": 76694, "rss_mb": 177.305, "state": "R (running)", "threads": 106}, "restart_count": 179, "restart_count_per_hour": 10, "rtsp_timeout_count": 290, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 180}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.885, "sample_count": 2206, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1421726, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:46:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.791, "last_frame_pushed_at": "2026-07-01T03:46:15+09:00", "last_frame_read_age_sec": 4.879, "last_frame_read_at": "2026-07-01T03:46:15+09:00", "mjpeg": {"age_sec": 4.775, "exists": true, "mtime": "2026-07-01T03:46:15.461555+09:00", "mtime_epoch": 1782845175.4615545, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105896}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T03:46:20+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1731.645, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 35, "pushed_frames": 35, "read_frames": 34, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.169, "pushed": 1.169, "read": 1.136}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.933, "count": 35, "max_ms": 26.8, "p95_ms": 21.952}, "gstreamer_push": {"avg_ms": 1.215, "count": 35, "max_ms": 1.394, "p95_ms": 1.328}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.375, "count": 35, "max_ms": 48.324, "p95_ms": 41.005}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.012, "count": 35, "max_ms": 48.837, "p95_ms": 48.53}, "yolo11_640_fire_11class": {"avg_ms": 46.605, "count": 35, "max_ms": 49.763, "p95_ms": 48.767}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.868, "count": 35, "max_ms": 49.434, "p95_ms": 49.341}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.886, "count": 35, "max_ms": 85.843, "p95_ms": 79.247}, "read_interval": {"avg_ms": 739.386, "count": 34, "max_ms": 13017.407, "p95_ms": 9218.558}, "triton_wait": {"avg_ms": 47.942, "count": 35, "max_ms": 49.777, "p95_ms": 49.445}, "visualize": {"avg_ms": 3.465, "count": 35, "max_ms": 5.254, "p95_ms": 3.714}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10508.809, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5565f1e4d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5565f1e4d440] error while decoding MB 0 34, bytestream 7230 [h264 @ 0x5565f1e49dc0] reference picture missing during reorder [h264 @ 0x5565f1e49dc0] Missing reference picture, default is 65540 [h264 @ 0x5565f1df6140] mmco: unref short failure [h264 @ 0x5565f1df6140] illegal short term buffer state detected [in#0/rtsp @ 0x5565f1dc6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5565f1dc6e00] Er", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 373, "h264_decode_error_count": 1229, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:47:09+09:00", "last_error_summary": "[h264 @ 0x5565f1e4d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5565f1e4d440] error while decoding MB 0 34, bytestream 7230 [h264 @ 0x5565f1e49dc0] reference picture missing during reorder [h264 @ 0x5565f1e49dc0] Missing reference picture, default is 65540 [h264 @ 0x5565f1df6140] mmco: unref short failure [h264 @ 0x5565f1df6140] illegal short term buffer state detected [in#0/rtsp @ 0x5565f1dc6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5565f1dc6e00] Er", "last_exit_at": "2026-07-01T03:42:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 264.856, "last_started_at": "2026-07-01T03:42:44+09:00", "pid": 76587, "process": {"exists": false, "pid": 76587}, "restart_count": 406, "restart_count_per_hour": 8, "rtsp_timeout_count": 364, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 407}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.498, "sample_count": 2350, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1284834, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:47:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.105, "last_frame_pushed_at": "2026-07-01T03:47:09+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T03:47:09+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-01T03:47:09.306501+09:00", "mtime_epoch": 1782845229.3065012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129551}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:47:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4977.469, "name": "celery", "pid": 267, "rss_mb": 4977.469, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 105, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.501, "pushed": 8.501, "read": 8.583}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.395, "count": 104, "max_ms": 24.02, "p95_ms": 23.176}, "gstreamer_push": {"avg_ms": 1.217, "count": 104, "max_ms": 1.392, "p95_ms": 1.297}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.9, "count": 104, "max_ms": 43.836, "p95_ms": 42.015}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.966, "count": 104, "max_ms": 75.816, "p95_ms": 68.682}, "yolo11_640_fire_11class": {"avg_ms": 62.941, "count": 104, "max_ms": 76.316, "p95_ms": 67.385}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.51, "count": 104, "max_ms": 86.239, "p95_ms": 77.89}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.069, "count": 104, "max_ms": 115.457, "p95_ms": 107.648}, "read_interval": {"avg_ms": 308.701, "count": 105, "max_ms": 20275.319, "p95_ms": 121.787}, "triton_wait": {"avg_ms": 73.522, "count": 104, "max_ms": 86.25, "p95_ms": 77.902}, "visualize": {"avg_ms": 3.399, "count": 104, "max_ms": 4.319, "p95_ms": 3.663}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 12.234, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10508.359, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:47:09+09:00", "last_error_summary": "[h264 @ 0x5565f1e4d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5565f1e4d440] error while decoding MB 0 34, bytestream 7230 [h264 @ 0x5565f1e49dc0] reference picture missing during reorder [h264 @ 0x5565f1e49dc0] Missing reference picture, default is 65540 [h264 @ 0x5565f1df6140] mmco: unref short failure [h264 @ 0x5565f1df6140] illegal short term buffer state detected [in#0/rtsp @ 0x5565f1dc6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5565f1dc6e00] Er", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 373, "h264_decode_error_count": 1229, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:47:09+09:00", "last_error_summary": "[h264 @ 0x5565f1e4d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5565f1e4d440] error while decoding MB 0 34, bytestream 7230 [h264 @ 0x5565f1e49dc0] reference picture missing during reorder [h264 @ 0x5565f1e49dc0] Missing reference picture, default is 65540 [h264 @ 0x5565f1df6140] mmco: unref short failure [h264 @ 0x5565f1df6140] illegal short term buffer state detected [in#0/rtsp @ 0x5565f1dc6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5565f1dc6e00] Er", "last_exit_at": "2026-07-01T03:47:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 264.882, "last_started_at": "2026-07-01T03:42:44+09:00", "pid": 76587, "process": {"exists": false, "pid": 76587}, "restart_count": 406, "restart_count_per_hour": 8, "rtsp_timeout_count": 364, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 407}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.518, "sample_count": 2350, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1284835, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:47:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T03:47:09+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-01T03:47:09+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-01T03:47:09.306501+09:00", "mtime_epoch": 1782845229.3065012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129551}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:47:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4977.469, "name": "celery", "pid": 267, "rss_mb": 4977.469, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.566, "pushed": 8.566, "read": 8.566}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.429, "count": 105, "max_ms": 25.049, "p95_ms": 23.231}, "gstreamer_push": {"avg_ms": 1.218, "count": 105, "max_ms": 1.392, "p95_ms": 1.306}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.995, "count": 105, "max_ms": 44.897, "p95_ms": 42.052}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.969, "count": 105, "max_ms": 75.816, "p95_ms": 68.682}, "yolo11_640_fire_11class": {"avg_ms": 62.954, "count": 105, "max_ms": 76.316, "p95_ms": 67.385}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.515, "count": 105, "max_ms": 86.239, "p95_ms": 77.89}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.131, "count": 105, "max_ms": 115.457, "p95_ms": 107.943}, "read_interval": {"avg_ms": 308.701, "count": 105, "max_ms": 20275.319, "p95_ms": 121.787}, "triton_wait": {"avg_ms": 73.527, "count": 105, "max_ms": 86.25, "p95_ms": 77.902}, "visualize": {"avg_ms": 3.419, "count": 105, "max_ms": 5.501, "p95_ms": 3.718}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 12.258, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10508.445, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 373, "h264_decode_error_count": 1229, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:47:09+09:00", "last_error_summary": "[h264 @ 0x5565f1e4d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5565f1e4d440] error while decoding MB 0 34, bytestream 7230 [h264 @ 0x5565f1e49dc0] reference picture missing during reorder [h264 @ 0x5565f1e49dc0] Missing reference picture, default is 65540 [h264 @ 0x5565f1df6140] mmco: unref short failure [h264 @ 0x5565f1df6140] illegal short term buffer state detected [in#0/rtsp @ 0x5565f1dc6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5565f1dc6e00] Er", "last_exit_at": "2026-07-01T03:47:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 264.892, "last_started_at": "2026-07-01T03:42:44+09:00", "pid": 76587, "process": {"exists": false, "pid": 76587}, "restart_count": 406, "restart_count_per_hour": 8, "rtsp_timeout_count": 364, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 407}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.518, "sample_count": 2350, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1284835, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:47:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-01T03:47:09+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-01T03:47:09+09:00", "mjpeg": {"age_sec": 0.124, "exists": true, "mtime": "2026-07-01T03:47:09.306501+09:00", "mtime_epoch": 1782845229.3065012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129551}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:47:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4977.469, "name": "celery", "pid": 267, "rss_mb": 4977.469, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.559, "pushed": 8.559, "read": 8.559}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.429, "count": 105, "max_ms": 25.049, "p95_ms": 23.231}, "gstreamer_push": {"avg_ms": 1.218, "count": 105, "max_ms": 1.392, "p95_ms": 1.306}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.995, "count": 105, "max_ms": 44.897, "p95_ms": 42.052}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.969, "count": 105, "max_ms": 75.816, "p95_ms": 68.682}, "yolo11_640_fire_11class": {"avg_ms": 62.954, "count": 105, "max_ms": 76.316, "p95_ms": 67.385}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.515, "count": 105, "max_ms": 86.239, "p95_ms": 77.89}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.131, "count": 105, "max_ms": 115.457, "p95_ms": 107.943}, "read_interval": {"avg_ms": 308.701, "count": 105, "max_ms": 20275.319, "p95_ms": 121.787}, "triton_wait": {"avg_ms": 73.527, "count": 105, "max_ms": 86.25, "p95_ms": 77.902}, "visualize": {"avg_ms": 3.419, "count": 105, "max_ms": 5.501, "p95_ms": 3.718}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 12.268, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10619.387, "limit_mb": 8796093022207.996}, "detail": {"pid": 76846, "restart_count": 407, "start_count": 408}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 373, "h264_decode_error_count": 1229, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:47:09+09:00", "last_error_summary": "[h264 @ 0x5565f1e4d440] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5565f1e4d440] error while decoding MB 0 34, bytestream 7230 [h264 @ 0x5565f1e49dc0] reference picture missing during reorder [h264 @ 0x5565f1e49dc0] Missing reference picture, default is 65540 [h264 @ 0x5565f1df6140] mmco: unref short failure [h264 @ 0x5565f1df6140] illegal short term buffer state detected [in#0/rtsp @ 0x5565f1dc6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5565f1dc6e00] Er", "last_exit_at": "2026-07-01T03:47:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:47:11+09:00", "pid": 76846, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.266, "name": "ffmpeg", "pid": 76846, "rss_mb": 141.266, "state": "S (sleeping)", "threads": 106}, "restart_count": 407, "restart_count_per_hour": 9, "rtsp_timeout_count": 364, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 408}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.53, "sample_count": 2303, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1284835, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:47:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.297, "last_frame_pushed_at": "2026-07-01T03:47:09+09:00", "last_frame_read_age_sec": 4.409, "last_frame_read_at": "2026-07-01T03:47:09+09:00", "mjpeg": {"age_sec": 4.279, "exists": true, "mtime": "2026-07-01T03:47:09.430503+09:00", "mtime_epoch": 1782845229.4305034, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 131706}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:47:13+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4977.469, "name": "celery", "pid": 267, "rss_mb": 4977.469, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.346, "pushed": 6.346, "read": 6.346}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.429, "count": 105, "max_ms": 25.049, "p95_ms": 23.231}, "gstreamer_push": {"avg_ms": 1.218, "count": 105, "max_ms": 1.392, "p95_ms": 1.306}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.995, "count": 105, "max_ms": 44.897, "p95_ms": 42.052}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.969, "count": 105, "max_ms": 75.816, "p95_ms": 68.682}, "yolo11_640_fire_11class": {"avg_ms": 62.954, "count": 105, "max_ms": 76.316, "p95_ms": 67.385}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.515, "count": 105, "max_ms": 86.239, "p95_ms": 77.89}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.131, "count": 105, "max_ms": 115.457, "p95_ms": 107.943}, "read_interval": {"avg_ms": 308.701, "count": 105, "max_ms": 20275.319, "p95_ms": 121.787}, "triton_wait": {"avg_ms": 73.527, "count": 105, "max_ms": 86.25, "p95_ms": 77.902}, "visualize": {"avg_ms": 3.419, "count": 105, "max_ms": 5.501, "p95_ms": 3.718}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.547, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10739}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10513.055, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56264e5f7040] left block unavailable for requested intra mode [h264 @ 0x56264e5f7040] error while decoding MB 0 30, bytestream 3583 [h264 @ 0x56264da5cc80] left block unavailable for requested intra mode [h264 @ 0x56264da5cc80] error while decoding MB 0 14, bytestream 1371 [h264 @ 0x56264da5cc80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56264da5cc80] error while decoding MB 0 10, bytestream 3140", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 374, "h264_decode_error_count": 1232, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:54:43+09:00", "last_error_summary": "[h264 @ 0x56264e5f7040] left block unavailable for requested intra mode [h264 @ 0x56264e5f7040] error while decoding MB 0 30, bytestream 3583 [h264 @ 0x56264da5cc80] left block unavailable for requested intra mode [h264 @ 0x56264da5cc80] error while decoding MB 0 14, bytestream 1371 [h264 @ 0x56264da5cc80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56264da5cc80] error while decoding MB 0 10, bytestream 3140", "last_exit_at": "2026-07-01T03:47:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 451.354, "last_started_at": "2026-07-01T03:47:11+09:00", "pid": 76846, "process": {"exists": false, "pid": 76846}, "restart_count": 407, "restart_count_per_hour": 8, "rtsp_timeout_count": 364, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 408}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.047, "sample_count": 2605, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1288783, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:54:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-01T03:54:42+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T03:54:43+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-01T03:54:43.014517+09:00", "mtime_epoch": 1782845683.0145168, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:54:43+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4979.273, "name": "celery", "pid": 267, "rss_mb": 4979.273, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 261, "pushed_frames": 261, "read_frames": 261, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.722, "pushed": 8.722, "read": 8.722}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.217, "count": 261, "max_ms": 22.957, "p95_ms": 22.066}, "gstreamer_push": {"avg_ms": 1.205, "count": 261, "max_ms": 1.389, "p95_ms": 1.282}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.996, "count": 261, "max_ms": 45.062, "p95_ms": 43.159}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.1, "count": 261, "max_ms": 65.881, "p95_ms": 64.189}, "yolo11_640_fire_11class": {"avg_ms": 60.061, "count": 261, "max_ms": 65.326, "p95_ms": 63.005}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.37, "count": 261, "max_ms": 77.501, "p95_ms": 74.427}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.822, "count": 261, "max_ms": 107.658, "p95_ms": 104.238}, "read_interval": {"avg_ms": 114.816, "count": 261, "max_ms": 123.025, "p95_ms": 120.03}, "triton_wait": {"avg_ms": 70.383, "count": 261, "max_ms": 77.513, "p95_ms": 74.44}, "visualize": {"avg_ms": 3.352, "count": 261, "max_ms": 4.086, "p95_ms": 3.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10739}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10512.68, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T03:54:43+09:00", "last_error_summary": "[h264 @ 0x56264e5f7040] left block unavailable for requested intra mode [h264 @ 0x56264e5f7040] error while decoding MB 0 30, bytestream 3583 [h264 @ 0x56264da5cc80] left block unavailable for requested intra mode [h264 @ 0x56264da5cc80] error while decoding MB 0 14, bytestream 1371 [h264 @ 0x56264da5cc80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56264da5cc80] error while decoding MB 0 10, bytestream 3140", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 374, "h264_decode_error_count": 1232, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:54:43+09:00", "last_error_summary": "[h264 @ 0x56264e5f7040] left block unavailable for requested intra mode [h264 @ 0x56264e5f7040] error while decoding MB 0 30, bytestream 3583 [h264 @ 0x56264da5cc80] left block unavailable for requested intra mode [h264 @ 0x56264da5cc80] error while decoding MB 0 14, bytestream 1371 [h264 @ 0x56264da5cc80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56264da5cc80] error while decoding MB 0 10, bytestream 3140", "last_exit_at": "2026-07-01T03:54:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 451.412, "last_started_at": "2026-07-01T03:47:11+09:00", "pid": 76846, "process": {"exists": false, "pid": 76846}, "restart_count": 407, "restart_count_per_hour": 8, "rtsp_timeout_count": 364, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 408}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.047, "sample_count": 2605, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1288784, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:54:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T03:54:43+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-01T03:54:43+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T03:54:43.014517+09:00", "mtime_epoch": 1782845683.0145168, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:54:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4979.273, "name": "celery", "pid": 267, "rss_mb": 4979.273, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 262, "pushed_frames": 262, "read_frames": 261, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.738, "pushed": 8.738, "read": 8.705}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.214, "count": 262, "max_ms": 22.957, "p95_ms": 22.066}, "gstreamer_push": {"avg_ms": 1.205, "count": 262, "max_ms": 1.389, "p95_ms": 1.282}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 37.004, "count": 262, "max_ms": 45.062, "p95_ms": 43.159}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.102, "count": 262, "max_ms": 65.881, "p95_ms": 64.189}, "yolo11_640_fire_11class": {"avg_ms": 60.064, "count": 262, "max_ms": 65.326, "p95_ms": 63.005}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.375, "count": 262, "max_ms": 77.501, "p95_ms": 74.427}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.823, "count": 262, "max_ms": 107.658, "p95_ms": 104.238}, "read_interval": {"avg_ms": 114.816, "count": 261, "max_ms": 123.025, "p95_ms": 120.03}, "triton_wait": {"avg_ms": 70.387, "count": 262, "max_ms": 77.513, "p95_ms": 74.44}, "visualize": {"avg_ms": 3.352, "count": 262, "max_ms": 4.086, "p95_ms": 3.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10739}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10512.699, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 374, "h264_decode_error_count": 1232, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:54:43+09:00", "last_error_summary": "[h264 @ 0x56264e5f7040] left block unavailable for requested intra mode [h264 @ 0x56264e5f7040] error while decoding MB 0 30, bytestream 3583 [h264 @ 0x56264da5cc80] left block unavailable for requested intra mode [h264 @ 0x56264da5cc80] error while decoding MB 0 14, bytestream 1371 [h264 @ 0x56264da5cc80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56264da5cc80] error while decoding MB 0 10, bytestream 3140", "last_exit_at": "2026-07-01T03:54:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 451.42, "last_started_at": "2026-07-01T03:47:11+09:00", "pid": 76846, "process": {"exists": false, "pid": 76846}, "restart_count": 407, "restart_count_per_hour": 8, "rtsp_timeout_count": 364, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 408}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.047, "sample_count": 2605, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1288784, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:54:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-01T03:54:43+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-01T03:54:43+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-01T03:54:43.014517+09:00", "mtime_epoch": 1782845683.0145168, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:54:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4979.531, "name": "celery", "pid": 267, "rss_mb": 4979.531, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 262, "pushed_frames": 262, "read_frames": 261, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.736, "pushed": 8.736, "read": 8.703}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.214, "count": 262, "max_ms": 22.957, "p95_ms": 22.066}, "gstreamer_push": {"avg_ms": 1.205, "count": 262, "max_ms": 1.389, "p95_ms": 1.282}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 37.004, "count": 262, "max_ms": 45.062, "p95_ms": 43.159}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.102, "count": 262, "max_ms": 65.881, "p95_ms": 64.189}, "yolo11_640_fire_11class": {"avg_ms": 60.064, "count": 262, "max_ms": 65.326, "p95_ms": 63.005}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.375, "count": 262, "max_ms": 77.501, "p95_ms": 74.427}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.823, "count": 262, "max_ms": 107.658, "p95_ms": 104.238}, "read_interval": {"avg_ms": 114.816, "count": 261, "max_ms": 123.025, "p95_ms": 120.03}, "triton_wait": {"avg_ms": 70.387, "count": 262, "max_ms": 77.513, "p95_ms": 74.44}, "visualize": {"avg_ms": 3.352, "count": 262, "max_ms": 4.086, "p95_ms": 3.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 410, "tracking_summary": 10739}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10620.668, "limit_mb": 8796093022207.996}, "detail": {"pid": 76953, "restart_count": 408, "start_count": 409}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 374, "h264_decode_error_count": 1232, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T03:54:43+09:00", "last_error_summary": "[h264 @ 0x56264e5f7040] left block unavailable for requested intra mode [h264 @ 0x56264e5f7040] error while decoding MB 0 30, bytestream 3583 [h264 @ 0x56264da5cc80] left block unavailable for requested intra mode [h264 @ 0x56264da5cc80] error while decoding MB 0 14, bytestream 1371 [h264 @ 0x56264da5cc80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56264da5cc80] error while decoding MB 0 10, bytestream 3140", "last_exit_at": "2026-07-01T03:54:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T03:54:45+09:00", "pid": 76953, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.594, "name": "ffmpeg", "pid": 76953, "rss_mb": 138.594, "state": "S (sleeping)", "threads": 106}, "restart_count": 408, "restart_count_per_hour": 9, "rtsp_timeout_count": 364, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 409}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.059, "sample_count": 2559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1288784, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T03:54:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.341, "last_frame_pushed_at": "2026-07-01T03:54:43+09:00", "last_frame_read_age_sec": 4.442, "last_frame_read_at": "2026-07-01T03:54:43+09:00", "mjpeg": {"age_sec": 4.326, "exists": true, "mtime": "2026-07-01T03:54:43.126519+09:00", "mtime_epoch": 1782845683.1265187, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121763}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T03:54:47+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4979.531, "name": "celery", "pid": 267, "rss_mb": 4979.531, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.455, "pushed": 7.455, "read": 7.422}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.196, "count": 223, "max_ms": 22.957, "p95_ms": 22.026}, "gstreamer_push": {"avg_ms": 1.201, "count": 223, "max_ms": 1.389, "p95_ms": 1.277}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 37.203, "count": 223, "max_ms": 45.062, "p95_ms": 43.167}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.301, "count": 223, "max_ms": 65.731, "p95_ms": 64.189}, "yolo11_640_fire_11class": {"avg_ms": 60.243, "count": 223, "max_ms": 64.896, "p95_ms": 63.039}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.738, "count": 223, "max_ms": 76.962, "p95_ms": 74.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.157, "count": 223, "max_ms": 106.666, "p95_ms": 104.27}, "read_interval": {"avg_ms": 115.14, "count": 222, "max_ms": 123.025, "p95_ms": 120.03}, "triton_wait": {"avg_ms": 70.751, "count": 223, "max_ms": 76.975, "p95_ms": 74.537}, "visualize": {"avg_ms": 3.345, "count": 223, "max_ms": 4.086, "p95_ms": 3.535}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10493.43, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55fc5abad5c0] error while decoding MB 2 49, bytestream -5 [in#0/rtsp @ 0x55fc5ab6be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc5ab6be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 180, "h264_decode_error_count": 185, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:06:10+09:00", "last_error_summary": "[h264 @ 0x55fc5abad5c0] error while decoding MB 2 49, bytestream -5 [in#0/rtsp @ 0x55fc5ab6be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc5ab6be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:46:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 1192.519, "last_started_at": "2026-07-01T03:46:18+09:00", "pid": 76694, "process": {"exists": false, "pid": 76694}, "restart_count": 179, "restart_count_per_hour": 8, "rtsp_timeout_count": 292, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 180}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.833, "sample_count": 2793, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1432777, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:06:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-01T04:06:10+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-01T04:06:10+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-01T04:06:10.698732+09:00", "mtime_epoch": 1782846370.6987324, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101040}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:06:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1737.059, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.721, "pushed": 0.721, "read": 0.777}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 29.884, "count": 13, "max_ms": 32.139, "p95_ms": 32.139}, "gstreamer_push": {"avg_ms": 1.902, "count": 13, "max_ms": 2.207, "p95_ms": 2.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.918, "count": 13, "max_ms": 66.585, "p95_ms": 66.585}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.537, "count": 13, "max_ms": 67.448, "p95_ms": 67.448}, "yolo11_640_fire_11class": {"avg_ms": 66.734, "count": 13, "max_ms": 71.586, "p95_ms": 71.586}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.984, "count": 13, "max_ms": 75.256, "p95_ms": 75.256}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.697, "count": 13, "max_ms": 119.706, "p95_ms": 119.706}, "read_interval": {"avg_ms": 2174.829, "count": 14, "max_ms": 16402.466, "p95_ms": 16402.466}, "triton_wait": {"avg_ms": 71.081, "count": 13, "max_ms": 75.271, "p95_ms": 75.271}, "visualize": {"avg_ms": 4.693, "count": 13, "max_ms": 5.346, "p95_ms": 5.346}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.029, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10493.117, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:06:10+09:00", "last_error_summary": "[h264 @ 0x55fc5abad5c0] error while decoding MB 2 49, bytestream -5 [in#0/rtsp @ 0x55fc5ab6be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc5ab6be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 180, "h264_decode_error_count": 185, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:06:10+09:00", "last_error_summary": "[h264 @ 0x55fc5abad5c0] error while decoding MB 2 49, bytestream -5 [in#0/rtsp @ 0x55fc5ab6be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc5ab6be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:06:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 1192.564, "last_started_at": "2026-07-01T03:46:18+09:00", "pid": 76694, "process": {"exists": false, "pid": 76694}, "restart_count": 179, "restart_count_per_hour": 8, "rtsp_timeout_count": 292, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 180}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.833, "sample_count": 2793, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1432778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:06:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T04:06:10+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-01T04:06:10+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-01T04:06:10.698732+09:00", "mtime_epoch": 1782846370.6987324, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101040}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:06:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1737.059, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.775, "pushed": 0.775, "read": 0.775}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 29.018, "count": 14, "max_ms": 32.139, "p95_ms": 32.139}, "gstreamer_push": {"avg_ms": 1.849, "count": 14, "max_ms": 2.207, "p95_ms": 2.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.458, "count": 14, "max_ms": 66.585, "p95_ms": 66.585}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.903, "count": 14, "max_ms": 69.662, "p95_ms": 69.662}, "yolo11_640_fire_11class": {"avg_ms": 67.009, "count": 14, "max_ms": 71.586, "p95_ms": 71.586}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.89, "count": 14, "max_ms": 75.256, "p95_ms": 75.256}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.594, "count": 14, "max_ms": 119.706, "p95_ms": 119.706}, "read_interval": {"avg_ms": 2174.829, "count": 14, "max_ms": 16402.466, "p95_ms": 16402.466}, "triton_wait": {"avg_ms": 71.047, "count": 14, "max_ms": 75.271, "p95_ms": 75.271}, "visualize": {"avg_ms": 4.607, "count": 14, "max_ms": 5.346, "p95_ms": 5.346}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.073, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10493.176, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 180, "h264_decode_error_count": 185, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:06:10+09:00", "last_error_summary": "[h264 @ 0x55fc5abad5c0] error while decoding MB 2 49, bytestream -5 [in#0/rtsp @ 0x55fc5ab6be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc5ab6be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:06:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 1192.569, "last_started_at": "2026-07-01T03:46:18+09:00", "pid": 76694, "process": {"exists": false, "pid": 76694}, "restart_count": 179, "restart_count_per_hour": 8, "rtsp_timeout_count": 292, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 180}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.833, "sample_count": 2793, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1432778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:06:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T04:06:10+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-01T04:06:10+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T04:06:10.698732+09:00", "mtime_epoch": 1782846370.6987324, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101040}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:06:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1737.059, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.774, "pushed": 0.774, "read": 0.774}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 29.018, "count": 14, "max_ms": 32.139, "p95_ms": 32.139}, "gstreamer_push": {"avg_ms": 1.849, "count": 14, "max_ms": 2.207, "p95_ms": 2.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.458, "count": 14, "max_ms": 66.585, "p95_ms": 66.585}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.903, "count": 14, "max_ms": 69.662, "p95_ms": 69.662}, "yolo11_640_fire_11class": {"avg_ms": 67.009, "count": 14, "max_ms": 71.586, "p95_ms": 71.586}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.89, "count": 14, "max_ms": 75.256, "p95_ms": 75.256}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.594, "count": 14, "max_ms": 119.706, "p95_ms": 119.706}, "read_interval": {"avg_ms": 2174.829, "count": 14, "max_ms": 16402.466, "p95_ms": 16402.466}, "triton_wait": {"avg_ms": 71.047, "count": 14, "max_ms": 75.271, "p95_ms": 75.271}, "visualize": {"avg_ms": 4.607, "count": 14, "max_ms": 5.346, "p95_ms": 5.346}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.079, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10514.059, "limit_mb": 8796093022207.996}, "detail": {"pid": 77120, "restart_count": 180, "start_count": 181}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 180, "h264_decode_error_count": 185, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:06:10+09:00", "last_error_summary": "[h264 @ 0x55fc5abad5c0] error while decoding MB 2 49, bytestream -5 [in#0/rtsp @ 0x55fc5ab6be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fc5ab6be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:06:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:06:12+09:00", "pid": 77120, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.68, "name": "ffmpeg", "pid": 77120, "rss_mb": 57.68, "state": "S (sleeping)", "threads": 1}, "restart_count": 180, "restart_count_per_hour": 9, "rtsp_timeout_count": 292, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 181}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.856, "sample_count": 2749, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1432778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:06:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.174, "last_frame_pushed_at": "2026-07-01T04:06:10+09:00", "last_frame_read_age_sec": 4.272, "last_frame_read_at": "2026-07-01T04:06:10+09:00", "mjpeg": {"age_sec": 4.157, "exists": true, "mtime": "2026-07-01T04:06:10.810734+09:00", "mtime_epoch": 1782846370.8107343, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94797}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:06:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1737.059, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.629, "pushed": 0.629, "read": 0.629}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 29.018, "count": 14, "max_ms": 32.139, "p95_ms": 32.139}, "gstreamer_push": {"avg_ms": 1.849, "count": 14, "max_ms": 2.207, "p95_ms": 2.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.458, "count": 14, "max_ms": 66.585, "p95_ms": 66.585}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.903, "count": 14, "max_ms": 69.662, "p95_ms": 69.662}, "yolo11_640_fire_11class": {"avg_ms": 67.009, "count": 14, "max_ms": 71.586, "p95_ms": 71.586}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.89, "count": 14, "max_ms": 75.256, "p95_ms": 75.256}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.594, "count": 14, "max_ms": 119.706, "p95_ms": 119.706}, "read_interval": {"avg_ms": 2174.829, "count": 14, "max_ms": 16402.466, "p95_ms": 16402.466}, "triton_wait": {"avg_ms": 71.047, "count": 14, "max_ms": 75.271, "p95_ms": 75.271}, "visualize": {"avg_ms": 4.607, "count": 14, "max_ms": 5.346, "p95_ms": 5.346}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.242, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11949}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10497.477, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55641e2f68c0] error while decoding MB 90 35, bytestream -11 [in#0/rtsp @ 0x55641daf7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55641daf7e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 181, "h264_decode_error_count": 186, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:08:52+09:00", "last_error_summary": "[h264 @ 0x55641e2f68c0] error while decoding MB 90 35, bytestream -11 [in#0/rtsp @ 0x55641daf7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55641daf7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:06:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.105, "last_started_at": "2026-07-01T04:06:12+09:00", "pid": 77120, "process": {"exists": false, "pid": 77120}, "restart_count": 180, "restart_count_per_hour": 8, "rtsp_timeout_count": 294, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 181}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.965, "sample_count": 2231, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1433921, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:08:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-01T04:08:51+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-01T04:08:52+09:00", "mjpeg": {"age_sec": 0.065, "exists": true, "mtime": "2026-07-01T04:08:52.006932+09:00", "mtime_epoch": 1782846532.0069318, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92223}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:08:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1737.574, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.253, "pushed": 10.253, "read": 11.108}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.472, "count": 12, "max_ms": 21.184, "p95_ms": 21.184}, "gstreamer_push": {"avg_ms": 1.195, "count": 12, "max_ms": 1.286, "p95_ms": 1.286}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.188, "count": 12, "max_ms": 49.847, "p95_ms": 49.847}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.315, "count": 12, "max_ms": 54.574, "p95_ms": 54.574}, "yolo11_640_fire_11class": {"avg_ms": 49.439, "count": 12, "max_ms": 56.087, "p95_ms": 56.087}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.285, "count": 12, "max_ms": 56.075, "p95_ms": 56.075}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.77, "count": 12, "max_ms": 83.77, "p95_ms": 83.77}, "read_interval": {"avg_ms": 2397.642, "count": 13, "max_ms": 30066.149, "p95_ms": 30066.149}, "triton_wait": {"avg_ms": 50.298, "count": 12, "max_ms": 56.098, "p95_ms": 56.098}, "visualize": {"avg_ms": 3.411, "count": 12, "max_ms": 3.55, "p95_ms": 3.55}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.17, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11949}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10497.137, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:08:52+09:00", "last_error_summary": "[h264 @ 0x55641e2f68c0] error while decoding MB 90 35, bytestream -11 [in#0/rtsp @ 0x55641daf7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55641daf7e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 181, "h264_decode_error_count": 186, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:08:52+09:00", "last_error_summary": "[h264 @ 0x55641e2f68c0] error while decoding MB 90 35, bytestream -11 [in#0/rtsp @ 0x55641daf7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55641daf7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:08:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.129, "last_started_at": "2026-07-01T04:06:12+09:00", "pid": 77120, "process": {"exists": false, "pid": 77120}, "restart_count": 180, "restart_count_per_hour": 8, "rtsp_timeout_count": 294, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 181}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.965, "sample_count": 2231, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1433922, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:08:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T04:08:52+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-01T04:08:52+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-01T04:08:52.006932+09:00", "mtime_epoch": 1782846532.0069318, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92223}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:08:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1737.574, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.883, "pushed": 10.883, "read": 10.883}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.758, "count": 13, "max_ms": 24.191, "p95_ms": 24.191}, "gstreamer_push": {"avg_ms": 1.212, "count": 13, "max_ms": 1.42, "p95_ms": 1.42}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.12, "count": 13, "max_ms": 49.847, "p95_ms": 49.847}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.042, "count": 13, "max_ms": 54.574, "p95_ms": 54.574}, "yolo11_640_fire_11class": {"avg_ms": 49.229, "count": 13, "max_ms": 56.087, "p95_ms": 56.087}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.048, "count": 13, "max_ms": 56.075, "p95_ms": 56.075}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.129, "count": 13, "max_ms": 83.77, "p95_ms": 83.77}, "read_interval": {"avg_ms": 2397.642, "count": 13, "max_ms": 30066.149, "p95_ms": 30066.149}, "triton_wait": {"avg_ms": 50.061, "count": 13, "max_ms": 56.098, "p95_ms": 56.098}, "visualize": {"avg_ms": 3.676, "count": 13, "max_ms": 6.858, "p95_ms": 6.858}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.195, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11949}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10497.129, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 181, "h264_decode_error_count": 186, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:08:52+09:00", "last_error_summary": "[h264 @ 0x55641e2f68c0] error while decoding MB 90 35, bytestream -11 [in#0/rtsp @ 0x55641daf7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55641daf7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:08:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.136, "last_started_at": "2026-07-01T04:06:12+09:00", "pid": 77120, "process": {"exists": false, "pid": 77120}, "restart_count": 180, "restart_count_per_hour": 8, "rtsp_timeout_count": 294, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 181}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.965, "sample_count": 2231, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1433922, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:08:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T04:08:52+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-01T04:08:52+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-01T04:08:52.006932+09:00", "mtime_epoch": 1782846532.0069318, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92223}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:08:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1737.574, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.818, "pushed": 10.818, "read": 10.818}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.758, "count": 13, "max_ms": 24.191, "p95_ms": 24.191}, "gstreamer_push": {"avg_ms": 1.212, "count": 13, "max_ms": 1.42, "p95_ms": 1.42}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.12, "count": 13, "max_ms": 49.847, "p95_ms": 49.847}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.042, "count": 13, "max_ms": 54.574, "p95_ms": 54.574}, "yolo11_640_fire_11class": {"avg_ms": 49.229, "count": 13, "max_ms": 56.087, "p95_ms": 56.087}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.048, "count": 13, "max_ms": 56.075, "p95_ms": 56.075}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.129, "count": 13, "max_ms": 83.77, "p95_ms": 83.77}, "read_interval": {"avg_ms": 2397.642, "count": 13, "max_ms": 30066.149, "p95_ms": 30066.149}, "triton_wait": {"avg_ms": 50.061, "count": 13, "max_ms": 56.098, "p95_ms": 56.098}, "visualize": {"avg_ms": 3.676, "count": 13, "max_ms": 6.858, "p95_ms": 6.858}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.202, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11949}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10674.25, "limit_mb": 8796093022207.996}, "detail": {"pid": 77272, "restart_count": 181, "start_count": 182}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 181, "h264_decode_error_count": 186, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:08:52+09:00", "last_error_summary": "[h264 @ 0x55641e2f68c0] error while decoding MB 90 35, bytestream -11 [in#0/rtsp @ 0x55641daf7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55641daf7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:08:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:08:54+09:00", "pid": 77272, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.367, "name": "ffmpeg", "pid": 77272, "rss_mb": 206.367, "state": "S (sleeping)", "threads": 106}, "restart_count": 181, "restart_count_per_hour": 9, "rtsp_timeout_count": 294, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 182}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.993, "sample_count": 2185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1433922, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:08:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.342, "last_frame_pushed_at": "2026-07-01T04:08:52+09:00", "last_frame_read_age_sec": 4.428, "last_frame_read_at": "2026-07-01T04:08:52+09:00", "mjpeg": {"age_sec": 4.326, "exists": true, "mtime": "2026-07-01T04:08:52.106932+09:00", "mtime_epoch": 1782846532.106932, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92319}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:08:56+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1737.574, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.35, "pushed": 2.35, "read": 2.35}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.758, "count": 13, "max_ms": 24.191, "p95_ms": 24.191}, "gstreamer_push": {"avg_ms": 1.212, "count": 13, "max_ms": 1.42, "p95_ms": 1.42}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.12, "count": 13, "max_ms": 49.847, "p95_ms": 49.847}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.042, "count": 13, "max_ms": 54.574, "p95_ms": 54.574}, "yolo11_640_fire_11class": {"avg_ms": 49.229, "count": 13, "max_ms": 56.087, "p95_ms": 56.087}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.048, "count": 13, "max_ms": 56.075, "p95_ms": 56.075}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.129, "count": 13, "max_ms": 83.77, "p95_ms": 83.77}, "read_interval": {"avg_ms": 2397.642, "count": 13, "max_ms": 30066.149, "p95_ms": 30066.149}, "triton_wait": {"avg_ms": 50.061, "count": 13, "max_ms": 56.098, "p95_ms": 56.098}, "visualize": {"avg_ms": 3.676, "count": 13, "max_ms": 6.858, "p95_ms": 6.858}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.531, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10818}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10531.906, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555e00ed0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e00ed0880] error while decoding MB 0 43, bytestream 45276 [h264 @ 0x555e00ed0880] error while decoding MB 56 23, bytestream -7 [h264 @ 0x555e0120c1c0] error while decoding MB 52 5, bytestream -7 [in#0/rtsp @ 0x555e00529e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555e00529e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 375, "h264_decode_error_count": 1235, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:11:01+09:00", "last_error_summary": "[h264 @ 0x555e00ed0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e00ed0880] error while decoding MB 0 43, bytestream 45276 [h264 @ 0x555e00ed0880] error while decoding MB 56 23, bytestream -7 [h264 @ 0x555e0120c1c0] error while decoding MB 52 5, bytestream -7 [in#0/rtsp @ 0x555e00529e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555e00529e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T03:54:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 976.467, "last_started_at": "2026-07-01T03:54:45+09:00", "pid": 76953, "process": {"exists": false, "pid": 76953}, "restart_count": 408, "restart_count_per_hour": 7, "rtsp_timeout_count": 366, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 409}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.376, "sample_count": 2748, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1298245, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:11:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T04:11:01+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-01T04:11:01+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-01T04:11:01.859331+09:00", "mtime_epoch": 1782846661.859331, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 111079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:11:01+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4984.168, "name": "celery", "pid": 267, "rss_mb": 4984.168, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.837, "pushed": 3.837, "read": 3.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.704, "count": 115, "max_ms": 26.827, "p95_ms": 25.958}, "gstreamer_push": {"avg_ms": 1.271, "count": 115, "max_ms": 1.526, "p95_ms": 1.449}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.269, "count": 115, "max_ms": 49.395, "p95_ms": 37.715}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 45.747, "count": 115, "max_ms": 67.184, "p95_ms": 51.398}, "yolo11_640_fire_11class": {"avg_ms": 45.22, "count": 115, "max_ms": 66.474, "p95_ms": 50.975}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 46.733, "count": 115, "max_ms": 75.956, "p95_ms": 52.007}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.87, "count": 115, "max_ms": 103.832, "p95_ms": 83.264}, "read_interval": {"avg_ms": 260.701, "count": 115, "max_ms": 19333.406, "p95_ms": 100.965}, "triton_wait": {"avg_ms": 46.846, "count": 115, "max_ms": 75.971, "p95_ms": 52.019}, "visualize": {"avg_ms": 3.407, "count": 115, "max_ms": 4.059, "p95_ms": 3.647}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10818}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10531.703, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:11:01+09:00", "last_error_summary": "[h264 @ 0x555e00ed0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e00ed0880] error while decoding MB 0 43, bytestream 45276 [h264 @ 0x555e00ed0880] error while decoding MB 56 23, bytestream -7 [h264 @ 0x555e0120c1c0] error while decoding MB 52 5, bytestream -7 [in#0/rtsp @ 0x555e00529e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555e00529e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 375, "h264_decode_error_count": 1235, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:11:01+09:00", "last_error_summary": "[h264 @ 0x555e00ed0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e00ed0880] error while decoding MB 0 43, bytestream 45276 [h264 @ 0x555e00ed0880] error while decoding MB 56 23, bytestream -7 [h264 @ 0x555e0120c1c0] error while decoding MB 52 5, bytestream -7 [in#0/rtsp @ 0x555e00529e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555e00529e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:11:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 976.493, "last_started_at": "2026-07-01T03:54:45+09:00", "pid": 76953, "process": {"exists": false, "pid": 76953}, "restart_count": 408, "restart_count_per_hour": 7, "rtsp_timeout_count": 366, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 409}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.376, "sample_count": 2749, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1298246, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:11:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T04:11:01+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-01T04:11:01+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-01T04:11:01.859331+09:00", "mtime_epoch": 1782846661.859331, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 111079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:11:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4984.168, "name": "celery", "pid": 267, "rss_mb": 4984.168, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.867, "pushed": 3.867, "read": 3.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.746, "count": 115, "max_ms": 26.827, "p95_ms": 25.958}, "gstreamer_push": {"avg_ms": 1.273, "count": 116, "max_ms": 1.526, "p95_ms": 1.449}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.276, "count": 115, "max_ms": 49.395, "p95_ms": 37.715}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 45.747, "count": 115, "max_ms": 67.184, "p95_ms": 51.398}, "yolo11_640_fire_11class": {"avg_ms": 45.219, "count": 115, "max_ms": 66.474, "p95_ms": 50.975}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 46.729, "count": 115, "max_ms": 75.956, "p95_ms": 52.007}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.907, "count": 116, "max_ms": 103.832, "p95_ms": 83.264}, "read_interval": {"avg_ms": 260.701, "count": 115, "max_ms": 19333.406, "p95_ms": 100.965}, "triton_wait": {"avg_ms": 46.841, "count": 115, "max_ms": 75.971, "p95_ms": 52.019}, "visualize": {"avg_ms": 3.432, "count": 115, "max_ms": 6.24, "p95_ms": 3.705}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10818}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10531.449, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 375, "h264_decode_error_count": 1235, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:11:01+09:00", "last_error_summary": "[h264 @ 0x555e00ed0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e00ed0880] error while decoding MB 0 43, bytestream 45276 [h264 @ 0x555e00ed0880] error while decoding MB 56 23, bytestream -7 [h264 @ 0x555e0120c1c0] error while decoding MB 52 5, bytestream -7 [in#0/rtsp @ 0x555e00529e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555e00529e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:11:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 976.502, "last_started_at": "2026-07-01T03:54:45+09:00", "pid": 76953, "process": {"exists": false, "pid": 76953}, "restart_count": 408, "restart_count_per_hour": 7, "rtsp_timeout_count": 366, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 409}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.376, "sample_count": 2749, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1298246, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:11:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-01T04:11:01+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T04:11:01+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-01T04:11:01.859331+09:00", "mtime_epoch": 1782846661.859331, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 111079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:11:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4984.168, "name": "celery", "pid": 267, "rss_mb": 4984.168, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.835, "pushed": 3.835, "read": 3.835}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.746, "count": 115, "max_ms": 26.827, "p95_ms": 25.958}, "gstreamer_push": {"avg_ms": 1.271, "count": 115, "max_ms": 1.526, "p95_ms": 1.419}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.276, "count": 115, "max_ms": 49.395, "p95_ms": 37.715}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 45.747, "count": 115, "max_ms": 67.184, "p95_ms": 51.398}, "yolo11_640_fire_11class": {"avg_ms": 45.219, "count": 115, "max_ms": 66.474, "p95_ms": 50.975}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 46.729, "count": 115, "max_ms": 75.956, "p95_ms": 52.007}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.936, "count": 115, "max_ms": 103.832, "p95_ms": 83.264}, "read_interval": {"avg_ms": 260.701, "count": 115, "max_ms": 19333.406, "p95_ms": 100.965}, "triton_wait": {"avg_ms": 46.841, "count": 115, "max_ms": 75.971, "p95_ms": 52.019}, "visualize": {"avg_ms": 3.432, "count": 115, "max_ms": 6.24, "p95_ms": 3.705}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10818}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10640.043, "limit_mb": 8796093022207.996}, "detail": {"pid": 77424, "restart_count": 409, "start_count": 410}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 375, "h264_decode_error_count": 1235, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:11:01+09:00", "last_error_summary": "[h264 @ 0x555e00ed0880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e00ed0880] error while decoding MB 0 43, bytestream 45276 [h264 @ 0x555e00ed0880] error while decoding MB 56 23, bytestream -7 [h264 @ 0x555e0120c1c0] error while decoding MB 52 5, bytestream -7 [in#0/rtsp @ 0x555e00529e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555e00529e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:11:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:11:03+09:00", "pid": 77424, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.258, "name": "ffmpeg", "pid": 77424, "rss_mb": 139.258, "state": "S (sleeping)", "threads": 106}, "restart_count": 409, "restart_count_per_hour": 8, "rtsp_timeout_count": 366, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 410}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.42, "sample_count": 2704, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1298246, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:11:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.058, "last_frame_pushed_at": "2026-07-01T04:11:01+09:00", "last_frame_read_age_sec": 4.142, "last_frame_read_at": "2026-07-01T04:11:01+09:00", "mjpeg": {"age_sec": 4.04, "exists": true, "mtime": "2026-07-01T04:11:01.955331+09:00", "mtime_epoch": 1782846661.9553313, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 107822}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:11:06+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4984.168, "name": "celery", "pid": 267, "rss_mb": 4984.168, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.403, "pushed": 2.403, "read": 2.37}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.288, "count": 72, "max_ms": 26.827, "p95_ms": 24.586}, "gstreamer_push": {"avg_ms": 1.281, "count": 72, "max_ms": 1.526, "p95_ms": 1.461}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.395, "count": 72, "max_ms": 41.92, "p95_ms": 37.705}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.357, "count": 72, "max_ms": 67.184, "p95_ms": 54.556}, "yolo11_640_fire_11class": {"avg_ms": 45.777, "count": 72, "max_ms": 66.474, "p95_ms": 53.56}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.499, "count": 72, "max_ms": 75.956, "p95_ms": 58.737}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.316, "count": 72, "max_ms": 103.832, "p95_ms": 87.866}, "read_interval": {"avg_ms": 364.673, "count": 71, "max_ms": 19333.406, "p95_ms": 111.628}, "triton_wait": {"avg_ms": 47.59, "count": 72, "max_ms": 75.971, "p95_ms": 58.747}, "visualize": {"avg_ms": 3.451, "count": 72, "max_ms": 6.24, "p95_ms": 3.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10828}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10533.504, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5602ede09980] error while decoding MB 16 32, bytestream -6 [in#0/rtsp @ 0x5602edd99e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5602edd99e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 376, "h264_decode_error_count": 1236, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:13:44+09:00", "last_error_summary": "[h264 @ 0x5602ede09980] error while decoding MB 16 32, bytestream -6 [in#0/rtsp @ 0x5602edd99e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5602edd99e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:11:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.902, "last_started_at": "2026-07-01T04:11:03+09:00", "pid": 77424, "process": {"exists": false, "pid": 77424}, "restart_count": 409, "restart_count_per_hour": 7, "rtsp_timeout_count": 368, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 410}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.101, "sample_count": 2410, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1299454, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:13:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T04:13:44+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T04:13:44+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-01T04:13:44.832333+09:00", "mtime_epoch": 1782846824.8323328, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119290}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:13:44+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4984.684, "name": "celery", "pid": 267, "rss_mb": 4984.684, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.341, "pushed": 3.341, "read": 3.341}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.488, "count": 100, "max_ms": 25.319, "p95_ms": 21.437}, "gstreamer_push": {"avg_ms": 1.191, "count": 100, "max_ms": 1.482, "p95_ms": 1.266}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.143, "count": 100, "max_ms": 42.182, "p95_ms": 39.097}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.897, "count": 100, "max_ms": 66.826, "p95_ms": 64.69}, "yolo11_640_fire_11class": {"avg_ms": 56.824, "count": 100, "max_ms": 65.863, "p95_ms": 63.486}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.101, "count": 100, "max_ms": 75.829, "p95_ms": 73.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.695, "count": 100, "max_ms": 104.549, "p95_ms": 101.915}, "read_interval": {"avg_ms": 299.479, "count": 100, "max_ms": 10139.822, "p95_ms": 118.526}, "triton_wait": {"avg_ms": 66.116, "count": 100, "max_ms": 75.844, "p95_ms": 73.888}, "visualize": {"avg_ms": 3.406, "count": 100, "max_ms": 3.723, "p95_ms": 3.544}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10828}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10533.445, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:13:44+09:00", "last_error_summary": "[h264 @ 0x5602ede09980] error while decoding MB 16 32, bytestream -6 [in#0/rtsp @ 0x5602edd99e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5602edd99e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 376, "h264_decode_error_count": 1236, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:13:44+09:00", "last_error_summary": "[h264 @ 0x5602ede09980] error while decoding MB 16 32, bytestream -6 [in#0/rtsp @ 0x5602edd99e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5602edd99e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:13:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.927, "last_started_at": "2026-07-01T04:11:03+09:00", "pid": 77424, "process": {"exists": false, "pid": 77424}, "restart_count": 409, "restart_count_per_hour": 7, "rtsp_timeout_count": 368, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 410}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.101, "sample_count": 2411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1299455, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:13:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T04:13:44+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-01T04:13:44+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-01T04:13:44.832333+09:00", "mtime_epoch": 1782846824.8323328, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119290}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:13:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4984.684, "name": "celery", "pid": 267, "rss_mb": 4984.684, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.372, "pushed": 3.372, "read": 3.339}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.551, "count": 101, "max_ms": 26.83, "p95_ms": 21.493}, "gstreamer_push": {"avg_ms": 1.193, "count": 101, "max_ms": 1.482, "p95_ms": 1.282}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.1, "count": 101, "max_ms": 42.182, "p95_ms": 39.097}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.767, "count": 101, "max_ms": 66.826, "p95_ms": 64.69}, "yolo11_640_fire_11class": {"avg_ms": 56.694, "count": 101, "max_ms": 65.863, "p95_ms": 63.486}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.896, "count": 101, "max_ms": 75.829, "p95_ms": 73.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.575, "count": 101, "max_ms": 104.549, "p95_ms": 101.915}, "read_interval": {"avg_ms": 299.479, "count": 100, "max_ms": 10139.822, "p95_ms": 118.526}, "triton_wait": {"avg_ms": 65.91, "count": 101, "max_ms": 75.844, "p95_ms": 73.888}, "visualize": {"avg_ms": 3.425, "count": 101, "max_ms": 5.333, "p95_ms": 3.548}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10828}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10533.344, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 376, "h264_decode_error_count": 1236, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:13:44+09:00", "last_error_summary": "[h264 @ 0x5602ede09980] error while decoding MB 16 32, bytestream -6 [in#0/rtsp @ 0x5602edd99e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5602edd99e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:13:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.935, "last_started_at": "2026-07-01T04:11:03+09:00", "pid": 77424, "process": {"exists": false, "pid": 77424}, "restart_count": 409, "restart_count_per_hour": 7, "rtsp_timeout_count": 368, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 410}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.101, "sample_count": 2411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1299455, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:13:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T04:13:44+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-01T04:13:44+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-01T04:13:44.832333+09:00", "mtime_epoch": 1782846824.8323328, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119290}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:13:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4984.684, "name": "celery", "pid": 267, "rss_mb": 4984.684, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.371, "pushed": 3.371, "read": 3.338}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.551, "count": 101, "max_ms": 26.83, "p95_ms": 21.493}, "gstreamer_push": {"avg_ms": 1.193, "count": 101, "max_ms": 1.482, "p95_ms": 1.282}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.1, "count": 101, "max_ms": 42.182, "p95_ms": 39.097}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.767, "count": 101, "max_ms": 66.826, "p95_ms": 64.69}, "yolo11_640_fire_11class": {"avg_ms": 56.694, "count": 101, "max_ms": 65.863, "p95_ms": 63.486}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.896, "count": 101, "max_ms": 75.829, "p95_ms": 73.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.575, "count": 101, "max_ms": 104.549, "p95_ms": 101.915}, "read_interval": {"avg_ms": 299.479, "count": 100, "max_ms": 10139.822, "p95_ms": 118.526}, "triton_wait": {"avg_ms": 65.91, "count": 101, "max_ms": 75.844, "p95_ms": 73.888}, "visualize": {"avg_ms": 3.425, "count": 101, "max_ms": 5.333, "p95_ms": 3.548}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10828}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10644.164, "limit_mb": 8796093022207.996}, "detail": {"pid": 77531, "restart_count": 410, "start_count": 411}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 376, "h264_decode_error_count": 1236, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:13:44+09:00", "last_error_summary": "[h264 @ 0x5602ede09980] error while decoding MB 16 32, bytestream -6 [in#0/rtsp @ 0x5602edd99e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5602edd99e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:13:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:13:47+09:00", "pid": 77531, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.512, "name": "ffmpeg", "pid": 77531, "rss_mb": 141.512, "state": "S (sleeping)", "threads": 106}, "restart_count": 410, "restart_count_per_hour": 8, "rtsp_timeout_count": 368, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 411}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.142, "sample_count": 2364, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1299455, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:13:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.157, "last_frame_pushed_at": "2026-07-01T04:13:44+09:00", "last_frame_read_age_sec": 4.242, "last_frame_read_at": "2026-07-01T04:13:44+09:00", "mjpeg": {"age_sec": 4.144, "exists": true, "mtime": "2026-07-01T04:13:44.928334+09:00", "mtime_epoch": 1782846824.9283335, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119763}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:13:49+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4984.684, "name": "celery", "pid": 267, "rss_mb": 4984.684, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.136, "pushed": 2.136, "read": 2.102}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.615, "count": 64, "max_ms": 26.83, "p95_ms": 21.815}, "gstreamer_push": {"avg_ms": 1.197, "count": 64, "max_ms": 1.482, "p95_ms": 1.302}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.126, "count": 64, "max_ms": 42.182, "p95_ms": 39.097}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.12, "count": 64, "max_ms": 66.826, "p95_ms": 65.621}, "yolo11_640_fire_11class": {"avg_ms": 55.913, "count": 64, "max_ms": 65.799, "p95_ms": 64.128}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.827, "count": 64, "max_ms": 75.829, "p95_ms": 73.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.554, "count": 64, "max_ms": 104.549, "p95_ms": 101.915}, "read_interval": {"avg_ms": 409.973, "count": 63, "max_ms": 10139.822, "p95_ms": 119.102}, "triton_wait": {"avg_ms": 64.842, "count": 64, "max_ms": 75.844, "p95_ms": 73.888}, "visualize": {"avg_ms": 3.422, "count": 64, "max_ms": 5.333, "p95_ms": 3.544}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10473.062, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55ee6dd78200] missing picture in access unit with size 18 [h264 @ 0x55ee6e63bec0] no frame! [in#0/rtsp @ 0x55ee6dd73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ee6dd73e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ee6dd78840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 182, "h264_decode_error_count": 187, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:14:27+09:00", "last_error_summary": "[NULL @ 0x55ee6dd78200] missing picture in access unit with size 18 [h264 @ 0x55ee6e63bec0] no frame! [in#0/rtsp @ 0x55ee6dd73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ee6dd73e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ee6dd78840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T04:08:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 332.579, "last_started_at": "2026-07-01T04:08:54+09:00", "pid": 77272, "process": {"exists": false, "pid": 77272}, "restart_count": 181, "restart_count_per_hour": 8, "rtsp_timeout_count": 296, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 182}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.868, "sample_count": 2603, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1436815, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:14:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-01T04:14:26+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-01T04:14:26+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T04:14:26.948665+09:00", "mtime_epoch": 1782846866.9486647, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92760}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:14:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1738.863, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 42, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.405, "pushed": 1.405, "read": 1.405}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.372, "count": 42, "max_ms": 20.782, "p95_ms": 19.747}, "gstreamer_push": {"avg_ms": 1.248, "count": 42, "max_ms": 1.43, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.413, "count": 42, "max_ms": 63.283, "p95_ms": 61.417}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.942, "count": 42, "max_ms": 68.477, "p95_ms": 68.148}, "yolo11_640_fire_11class": {"avg_ms": 61.94, "count": 42, "max_ms": 67.949, "p95_ms": 67.252}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.145, "count": 42, "max_ms": 78.827, "p95_ms": 77.561}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.818, "count": 42, "max_ms": 104.911, "p95_ms": 104.515}, "read_interval": {"avg_ms": 712.76, "count": 42, "max_ms": 25354.554, "p95_ms": 120.845}, "triton_wait": {"avg_ms": 70.255, "count": 42, "max_ms": 78.839, "p95_ms": 77.573}, "visualize": {"avg_ms": 3.408, "count": 42, "max_ms": 3.594, "p95_ms": 3.557}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10472.93, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:14:27+09:00", "last_error_summary": "[NULL @ 0x55ee6dd78200] missing picture in access unit with size 18 [h264 @ 0x55ee6e63bec0] no frame! [in#0/rtsp @ 0x55ee6dd73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ee6dd73e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ee6dd78840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 182, "h264_decode_error_count": 187, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:14:27+09:00", "last_error_summary": "[NULL @ 0x55ee6dd78200] missing picture in access unit with size 18 [h264 @ 0x55ee6e63bec0] no frame! [in#0/rtsp @ 0x55ee6dd73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ee6dd73e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ee6dd78840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T04:14:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 332.627, "last_started_at": "2026-07-01T04:08:54+09:00", "pid": 77272, "process": {"exists": false, "pid": 77272}, "restart_count": 181, "restart_count_per_hour": 8, "rtsp_timeout_count": 296, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 182}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.868, "sample_count": 2603, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1436816, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:14:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T04:14:27+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T04:14:26+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-01T04:14:26.948665+09:00", "mtime_epoch": 1782846866.9486647, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92760}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:14:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1738.863, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.436, "pushed": 1.436, "read": 1.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.409, "count": 43, "max_ms": 20.782, "p95_ms": 19.943}, "gstreamer_push": {"avg_ms": 1.249, "count": 43, "max_ms": 1.43, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.325, "count": 43, "max_ms": 63.283, "p95_ms": 61.417}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.088, "count": 43, "max_ms": 69.257, "p95_ms": 68.264}, "yolo11_640_fire_11class": {"avg_ms": 62.01, "count": 43, "max_ms": 67.949, "p95_ms": 67.252}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.246, "count": 43, "max_ms": 78.827, "p95_ms": 77.561}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.953, "count": 43, "max_ms": 104.911, "p95_ms": 104.515}, "read_interval": {"avg_ms": 712.76, "count": 42, "max_ms": 25354.554, "p95_ms": 120.845}, "triton_wait": {"avg_ms": 70.353, "count": 43, "max_ms": 78.839, "p95_ms": 77.573}, "visualize": {"avg_ms": 3.408, "count": 43, "max_ms": 3.594, "p95_ms": 3.557}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10472.828, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 182, "h264_decode_error_count": 187, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:14:27+09:00", "last_error_summary": "[NULL @ 0x55ee6dd78200] missing picture in access unit with size 18 [h264 @ 0x55ee6e63bec0] no frame! [in#0/rtsp @ 0x55ee6dd73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ee6dd73e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ee6dd78840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T04:14:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 332.633, "last_started_at": "2026-07-01T04:08:54+09:00", "pid": 77272, "process": {"exists": false, "pid": 77272}, "restart_count": 181, "restart_count_per_hour": 8, "rtsp_timeout_count": 296, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 182}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.868, "sample_count": 2603, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1436816, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:14:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T04:14:27+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-01T04:14:26+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-01T04:14:26.948665+09:00", "mtime_epoch": 1782846866.9486647, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92760}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:14:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1738.863, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.436, "pushed": 1.436, "read": 1.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.409, "count": 43, "max_ms": 20.782, "p95_ms": 19.943}, "gstreamer_push": {"avg_ms": 1.249, "count": 43, "max_ms": 1.43, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.325, "count": 43, "max_ms": 63.283, "p95_ms": 61.417}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.088, "count": 43, "max_ms": 69.257, "p95_ms": 68.264}, "yolo11_640_fire_11class": {"avg_ms": 62.01, "count": 43, "max_ms": 67.949, "p95_ms": 67.252}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.246, "count": 43, "max_ms": 78.827, "p95_ms": 77.561}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.953, "count": 43, "max_ms": 104.911, "p95_ms": 104.515}, "read_interval": {"avg_ms": 712.76, "count": 42, "max_ms": 25354.554, "p95_ms": 120.845}, "triton_wait": {"avg_ms": 70.353, "count": 43, "max_ms": 78.839, "p95_ms": 77.573}, "visualize": {"avg_ms": 3.408, "count": 43, "max_ms": 3.594, "p95_ms": 3.557}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10493.395, "limit_mb": 8796093022207.996}, "detail": {"pid": 77638, "restart_count": 182, "start_count": 183}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 182, "h264_decode_error_count": 187, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:14:27+09:00", "last_error_summary": "[NULL @ 0x55ee6dd78200] missing picture in access unit with size 18 [h264 @ 0x55ee6e63bec0] no frame! [in#0/rtsp @ 0x55ee6dd73e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ee6dd73e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ee6dd78840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T04:14:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:14:29+09:00", "pid": 77638, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.922, "name": "ffmpeg", "pid": 77638, "rss_mb": 57.922, "state": "S (sleeping)", "threads": 1}, "restart_count": 182, "restart_count_per_hour": 9, "rtsp_timeout_count": 296, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 183}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.789, "sample_count": 2564, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1436816, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:14:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.632, "last_frame_pushed_at": "2026-07-01T04:14:27+09:00", "last_frame_read_age_sec": 4.736, "last_frame_read_at": "2026-07-01T04:14:26+09:00", "mjpeg": {"age_sec": 4.617, "exists": true, "mtime": "2026-07-01T04:14:27.064666+09:00", "mtime_epoch": 1782846867.0646656, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93376}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:14:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1738.863, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.074, "pushed": 2.074, "read": 2.074}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.924, "count": 12, "max_ms": 20.782, "p95_ms": 20.782}, "gstreamer_push": {"avg_ms": 1.237, "count": 12, "max_ms": 1.43, "p95_ms": 1.43}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.448, "count": 12, "max_ms": 58.786, "p95_ms": 58.786}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.632, "count": 12, "max_ms": 69.257, "p95_ms": 69.257}, "yolo11_640_fire_11class": {"avg_ms": 52.918, "count": 12, "max_ms": 64.94, "p95_ms": 64.94}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.097, "count": 12, "max_ms": 74.485, "p95_ms": 74.485}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.64, "count": 12, "max_ms": 102.607, "p95_ms": 102.607}, "read_interval": {"avg_ms": 2200.349, "count": 12, "max_ms": 25354.554, "p95_ms": 25354.554}, "triton_wait": {"avg_ms": 56.45, "count": 12, "max_ms": 74.494, "p95_ms": 74.494}, "visualize": {"avg_ms": 3.393, "count": 12, "max_ms": 3.558, "p95_ms": 3.558}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.786, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11992}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10478.617, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x560345670cc0] error while decoding MB 71 36, bytestream -35 [in#0/rtsp @ 0x56034499ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56034499ee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 183, "h264_decode_error_count": 188, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:19:16+09:00", "last_error_summary": "[h264 @ 0x560345670cc0] error while decoding MB 71 36, bytestream -35 [in#0/rtsp @ 0x56034499ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56034499ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:14:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 286.865, "last_started_at": "2026-07-01T04:14:29+09:00", "pid": 77638, "process": {"exists": false, "pid": 77638}, "restart_count": 182, "restart_count_per_hour": 8, "rtsp_timeout_count": 298, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 183}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.614, "sample_count": 2292, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1439096, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:19:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-01T04:19:16+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T04:19:16+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T04:19:16.483534+09:00", "mtime_epoch": 1782847156.483534, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96685}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:19:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1740.152, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.268, "pushed": 3.268, "read": 3.268}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.739, "count": 98, "max_ms": 21.58, "p95_ms": 20.622}, "gstreamer_push": {"avg_ms": 1.235, "count": 98, "max_ms": 1.325, "p95_ms": 1.317}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.832, "count": 98, "max_ms": 66.166, "p95_ms": 58.21}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.753, "count": 98, "max_ms": 70.672, "p95_ms": 68.978}, "yolo11_640_fire_11class": {"avg_ms": 62.338, "count": 98, "max_ms": 67.618, "p95_ms": 66.959}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.413, "count": 98, "max_ms": 78.426, "p95_ms": 76.671}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.455, "count": 98, "max_ms": 106.096, "p95_ms": 104.257}, "read_interval": {"avg_ms": 306.377, "count": 98, "max_ms": 10166.452, "p95_ms": 120.445}, "triton_wait": {"avg_ms": 71.444, "count": 98, "max_ms": 78.44, "p95_ms": 76.684}, "visualize": {"avg_ms": 3.462, "count": 98, "max_ms": 4.296, "p95_ms": 3.703}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11992}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10478.383, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:19:16+09:00", "last_error_summary": "[h264 @ 0x560345670cc0] error while decoding MB 71 36, bytestream -35 [in#0/rtsp @ 0x56034499ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56034499ee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 183, "h264_decode_error_count": 188, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:19:16+09:00", "last_error_summary": "[h264 @ 0x560345670cc0] error while decoding MB 71 36, bytestream -35 [in#0/rtsp @ 0x56034499ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56034499ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:19:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 286.893, "last_started_at": "2026-07-01T04:14:29+09:00", "pid": 77638, "process": {"exists": false, "pid": 77638}, "restart_count": 182, "restart_count_per_hour": 8, "rtsp_timeout_count": 298, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 183}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.614, "sample_count": 2293, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1439097, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:19:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T04:19:16+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-01T04:19:16+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T04:19:16.483534+09:00", "mtime_epoch": 1782847156.483534, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96685}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:19:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1740.152, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.277, "pushed": 3.277, "read": 3.244}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.787, "count": 98, "max_ms": 24.639, "p95_ms": 21.014}, "gstreamer_push": {"avg_ms": 1.235, "count": 98, "max_ms": 1.325, "p95_ms": 1.317}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.617, "count": 98, "max_ms": 66.166, "p95_ms": 58.21}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.523, "count": 98, "max_ms": 70.672, "p95_ms": 68.978}, "yolo11_640_fire_11class": {"avg_ms": 62.113, "count": 98, "max_ms": 67.618, "p95_ms": 66.942}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.139, "count": 98, "max_ms": 78.426, "p95_ms": 76.671}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.253, "count": 98, "max_ms": 106.096, "p95_ms": 104.032}, "read_interval": {"avg_ms": 308.31, "count": 97, "max_ms": 10166.452, "p95_ms": 120.445}, "triton_wait": {"avg_ms": 71.169, "count": 98, "max_ms": 78.44, "p95_ms": 76.684}, "visualize": {"avg_ms": 3.486, "count": 98, "max_ms": 5.901, "p95_ms": 3.916}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11992}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10478.277, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 183, "h264_decode_error_count": 188, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:19:16+09:00", "last_error_summary": "[h264 @ 0x560345670cc0] error while decoding MB 71 36, bytestream -35 [in#0/rtsp @ 0x56034499ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56034499ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:19:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 286.901, "last_started_at": "2026-07-01T04:14:29+09:00", "pid": 77638, "process": {"exists": false, "pid": 77638}, "restart_count": 182, "restart_count_per_hour": 8, "rtsp_timeout_count": 298, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 183}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.614, "sample_count": 2293, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1439097, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:19:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-01T04:19:16+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T04:19:16+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-01T04:19:16.483534+09:00", "mtime_epoch": 1782847156.483534, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96685}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:19:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1740.152, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.276, "pushed": 3.276, "read": 3.243}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.787, "count": 98, "max_ms": 24.639, "p95_ms": 21.014}, "gstreamer_push": {"avg_ms": 1.235, "count": 98, "max_ms": 1.325, "p95_ms": 1.317}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.617, "count": 98, "max_ms": 66.166, "p95_ms": 58.21}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.523, "count": 98, "max_ms": 70.672, "p95_ms": 68.978}, "yolo11_640_fire_11class": {"avg_ms": 62.113, "count": 98, "max_ms": 67.618, "p95_ms": 66.942}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.139, "count": 98, "max_ms": 78.426, "p95_ms": 76.671}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.253, "count": 98, "max_ms": 106.096, "p95_ms": 104.032}, "read_interval": {"avg_ms": 308.31, "count": 97, "max_ms": 10166.452, "p95_ms": 120.445}, "triton_wait": {"avg_ms": 71.169, "count": 98, "max_ms": 78.44, "p95_ms": 76.684}, "visualize": {"avg_ms": 3.486, "count": 98, "max_ms": 5.901, "p95_ms": 3.916}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 11992}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10570.129, "limit_mb": 8796093022207.996}, "detail": {"pid": 77790, "restart_count": 183, "start_count": 184}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 183, "h264_decode_error_count": 188, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:19:16+09:00", "last_error_summary": "[h264 @ 0x560345670cc0] error while decoding MB 71 36, bytestream -35 [in#0/rtsp @ 0x56034499ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56034499ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:19:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:19:19+09:00", "pid": 77790, "process": {"exists": true, "fd_count": 5, "hwm_mb": 124.699, "name": "ffmpeg", "pid": 77790, "rss_mb": 124.699, "state": "S (sleeping)", "threads": 19}, "restart_count": 183, "restart_count_per_hour": 9, "rtsp_timeout_count": 298, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 184}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.614, "sample_count": 2293, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1439097, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:19:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.462, "last_frame_pushed_at": "2026-07-01T04:19:16+09:00", "last_frame_read_age_sec": 4.548, "last_frame_read_at": "2026-07-01T04:19:16+09:00", "mjpeg": {"age_sec": 4.447, "exists": true, "mtime": "2026-07-01T04:19:16.579535+09:00", "mtime_epoch": 1782847156.579535, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91649}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:19:21+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1740.152, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.005, "pushed": 2.005, "read": 1.972}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.83, "count": 60, "max_ms": 24.639, "p95_ms": 20.622}, "gstreamer_push": {"avg_ms": 1.231, "count": 60, "max_ms": 1.325, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.185, "count": 60, "max_ms": 59.053, "p95_ms": 56.271}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.893, "count": 60, "max_ms": 70.672, "p95_ms": 68.832}, "yolo11_640_fire_11class": {"avg_ms": 60.342, "count": 60, "max_ms": 67.618, "p95_ms": 66.829}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.939, "count": 60, "max_ms": 76.977, "p95_ms": 76.49}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.091, "count": 60, "max_ms": 104.257, "p95_ms": 103.562}, "read_interval": {"avg_ms": 431.762, "count": 59, "max_ms": 10166.452, "p95_ms": 120.841}, "triton_wait": {"avg_ms": 68.981, "count": 60, "max_ms": 76.989, "p95_ms": 76.503}, "visualize": {"avg_ms": 3.488, "count": 60, "max_ms": 5.901, "p95_ms": 3.666}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12018}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10785.844, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 183, "h264_decode_error_count": 188, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:19:16+09:00", "last_error_summary": "[h264 @ 0x560345670cc0] error while decoding MB 71 36, bytestream -35 [in#0/rtsp @ 0x56034499ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56034499ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:19:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 354.068, "last_started_at": "2026-07-01T04:19:19+09:00", "pid": 77790, "process": {"exists": true, "fd_count": 5, "hwm_mb": 328.902, "name": "ffmpeg", "pid": 77790, "rss_mb": 328.902, "state": "S (sleeping)", "threads": 106}, "restart_count": 183, "restart_count_per_hour": 8, "rtsp_timeout_count": 298, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 184}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.753, "sample_count": 2641, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1442243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:24:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.056, "last_frame_pushed_at": "2026-07-01T04:24:43+09:00", "last_frame_read_age_sec": 30.139, "last_frame_read_at": "2026-07-01T04:24:42+09:00", "mjpeg": {"age_sec": 30.038, "exists": true, "mtime": "2026-07-01T04:24:43.055591+09:00", "mtime_epoch": 1782847483.0555909, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98334}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:25:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1741.441, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12018}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10765.641, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 183, "h264_decode_error_count": 188, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:19:16+09:00", "last_error_summary": "[h264 @ 0x560345670cc0] error while decoding MB 71 36, bytestream -35 [in#0/rtsp @ 0x56034499ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56034499ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:19:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 354.082, "last_started_at": "2026-07-01T04:19:19+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 183, "restart_count_per_hour": 8, "rtsp_timeout_count": 298, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 184}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.753, "sample_count": 2641, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1442243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:24:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.069, "last_frame_pushed_at": "2026-07-01T04:24:43+09:00", "last_frame_read_age_sec": 30.152, "last_frame_read_at": "2026-07-01T04:24:42+09:00", "mjpeg": {"age_sec": 30.052, "exists": true, "mtime": "2026-07-01T04:24:43.055591+09:00", "mtime_epoch": 1782847483.0555909, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98334}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:25:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1741.441, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12018}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10484.84, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d28d44e800] error while decoding MB 6 37, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 184, "h264_decode_error_count": 189, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:25:13+09:00", "last_error_summary": "[h264 @ 0x55d28d44e800] error while decoding MB 6 37, bytestream -9", "last_exit_at": "2026-07-01T04:19:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 354.131, "last_started_at": "2026-07-01T04:19:19+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 183, "restart_count_per_hour": 8, "rtsp_timeout_count": 298, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 184}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.753, "sample_count": 2641, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1442243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:24:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.119, "last_frame_pushed_at": "2026-07-01T04:24:43+09:00", "last_frame_read_age_sec": 30.202, "last_frame_read_at": "2026-07-01T04:24:42+09:00", "mjpeg": {"age_sec": 30.101, "exists": true, "mtime": "2026-07-01T04:24:43.055591+09:00", "mtime_epoch": 1782847483.0555909, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98334}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:25:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1741.441, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12018}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10608.133, "limit_mb": 8796093022207.996}, "detail": {"pid": 77942, "restart_count": 184, "start_count": 185}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 184, "h264_decode_error_count": 189, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:25:13+09:00", "last_error_summary": "[h264 @ 0x55d28d44e800] error while decoding MB 6 37, bytestream -9", "last_exit_at": "2026-07-01T04:19:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:25:15+09:00", "pid": 77942, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.133, "name": "ffmpeg", "pid": 77942, "rss_mb": 156.133, "state": "R (running)", "threads": 49}, "restart_count": 184, "restart_count_per_hour": 9, "rtsp_timeout_count": 298, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 185}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.737, "sample_count": 2605, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1442243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:24:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 34.126, "last_frame_pushed_at": "2026-07-01T04:24:43+09:00", "last_frame_read_age_sec": 34.209, "last_frame_read_at": "2026-07-01T04:24:42+09:00", "mjpeg": {"age_sec": 34.109, "exists": true, "mtime": "2026-07-01T04:24:43.055591+09:00", "mtime_epoch": 1782847483.0555909, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98334}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:25:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1741.441, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10892}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10539.113, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563d65894080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d65894080] error while decoding MB 0 9, bytestream 5725 [h264 @ 0x563d661163c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d661163c0] error while decoding MB 0 35, bytestream 3571 [h264 @ 0x563d6596ad00] left block unavailable for requested intra mode [h264 @ 0x563d6596ad00] error while decoding MB 0 13, bytestream 5530 [in#0/rtsp @ 0x563d65824e00] Error during demuxing: Connection ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 377, "h264_decode_error_count": 1239, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:27:39+09:00", "last_error_summary": "[h264 @ 0x563d65894080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d65894080] error while decoding MB 0 9, bytestream 5725 [h264 @ 0x563d661163c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d661163c0] error while decoding MB 0 35, bytestream 3571 [h264 @ 0x563d6596ad00] left block unavailable for requested intra mode [h264 @ 0x563d6596ad00] error while decoding MB 0 13, bytestream 5530 [in#0/rtsp @ 0x563d65824e00] Error during demuxing: Connection ", "last_exit_at": "2026-07-01T04:13:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 832.729, "last_started_at": "2026-07-01T04:13:47+09:00", "pid": 77531, "process": {"exists": false, "pid": 77531}, "restart_count": 410, "restart_count_per_hour": 7, "rtsp_timeout_count": 370, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 411}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.943, "sample_count": 2728, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1307152, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:27:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-01T04:27:39+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-01T04:27:39+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T04:27:39.738000+09:00", "mtime_epoch": 1782847659.738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126202}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:27:39+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4988.551, "name": "celery", "pid": 267, "rss_mb": 4988.551, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 138, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.608, "pushed": 4.608, "read": 4.608}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.375, "count": 138, "max_ms": 31.743, "p95_ms": 25.22}, "gstreamer_push": {"avg_ms": 1.225, "count": 138, "max_ms": 1.498, "p95_ms": 1.333}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.331, "count": 138, "max_ms": 34.964, "p95_ms": 33.527}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.445, "count": 138, "max_ms": 49.224, "p95_ms": 48.3}, "yolo11_640_fire_11class": {"avg_ms": 43.752, "count": 138, "max_ms": 48.069, "p95_ms": 47.248}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.155, "count": 138, "max_ms": 49.767, "p95_ms": 48.855}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.432, "count": 138, "max_ms": 85.154, "p95_ms": 81.033}, "read_interval": {"avg_ms": 217.074, "count": 138, "max_ms": 10119.573, "p95_ms": 97.265}, "triton_wait": {"avg_ms": 45.205, "count": 138, "max_ms": 49.79, "p95_ms": 48.865}, "visualize": {"avg_ms": 3.336, "count": 138, "max_ms": 3.948, "p95_ms": 3.601}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10892}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10538.848, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:27:39+09:00", "last_error_summary": "[h264 @ 0x563d65894080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d65894080] error while decoding MB 0 9, bytestream 5725 [h264 @ 0x563d661163c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d661163c0] error while decoding MB 0 35, bytestream 3571 [h264 @ 0x563d6596ad00] left block unavailable for requested intra mode [h264 @ 0x563d6596ad00] error while decoding MB 0 13, bytestream 5530 [in#0/rtsp @ 0x563d65824e00] Error during demuxing: Connection ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 377, "h264_decode_error_count": 1239, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:27:39+09:00", "last_error_summary": "[h264 @ 0x563d65894080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d65894080] error while decoding MB 0 9, bytestream 5725 [h264 @ 0x563d661163c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d661163c0] error while decoding MB 0 35, bytestream 3571 [h264 @ 0x563d6596ad00] left block unavailable for requested intra mode [h264 @ 0x563d6596ad00] error while decoding MB 0 13, bytestream 5530 [in#0/rtsp @ 0x563d65824e00] Error during demuxing: Connection ", "last_exit_at": "2026-07-01T04:27:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 832.755, "last_started_at": "2026-07-01T04:13:47+09:00", "pid": 77531, "process": {"exists": false, "pid": 77531}, "restart_count": 410, "restart_count_per_hour": 7, "rtsp_timeout_count": 370, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 411}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.943, "sample_count": 2729, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1307153, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:27:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T04:27:39+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-01T04:27:39+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T04:27:39.738000+09:00", "mtime_epoch": 1782847659.738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126202}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:27:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4988.551, "name": "celery", "pid": 267, "rss_mb": 4988.551, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 138, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.638, "pushed": 4.638, "read": 4.604}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.383, "count": 139, "max_ms": 31.743, "p95_ms": 25.22}, "gstreamer_push": {"avg_ms": 1.226, "count": 139, "max_ms": 1.498, "p95_ms": 1.333}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.33, "count": 139, "max_ms": 34.964, "p95_ms": 33.527}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.464, "count": 139, "max_ms": 49.224, "p95_ms": 48.3}, "yolo11_640_fire_11class": {"avg_ms": 43.732, "count": 139, "max_ms": 48.069, "p95_ms": 47.248}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.145, "count": 139, "max_ms": 49.767, "p95_ms": 48.855}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.483, "count": 139, "max_ms": 85.154, "p95_ms": 81.12}, "read_interval": {"avg_ms": 217.074, "count": 138, "max_ms": 10119.573, "p95_ms": 97.265}, "triton_wait": {"avg_ms": 45.219, "count": 139, "max_ms": 49.79, "p95_ms": 48.865}, "visualize": {"avg_ms": 3.361, "count": 139, "max_ms": 6.776, "p95_ms": 3.643}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10892}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10538.965, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 377, "h264_decode_error_count": 1239, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:27:39+09:00", "last_error_summary": "[h264 @ 0x563d65894080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d65894080] error while decoding MB 0 9, bytestream 5725 [h264 @ 0x563d661163c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d661163c0] error while decoding MB 0 35, bytestream 3571 [h264 @ 0x563d6596ad00] left block unavailable for requested intra mode [h264 @ 0x563d6596ad00] error while decoding MB 0 13, bytestream 5530 [in#0/rtsp @ 0x563d65824e00] Error during demuxing: Connection ", "last_exit_at": "2026-07-01T04:27:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 832.764, "last_started_at": "2026-07-01T04:13:47+09:00", "pid": 77531, "process": {"exists": false, "pid": 77531}, "restart_count": 410, "restart_count_per_hour": 7, "rtsp_timeout_count": 370, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 411}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.943, "sample_count": 2729, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1307153, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:27:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-01T04:27:39+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T04:27:39+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-01T04:27:39.738000+09:00", "mtime_epoch": 1782847659.738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126202}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:27:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4988.551, "name": "celery", "pid": 267, "rss_mb": 4988.551, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 138, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.637, "pushed": 4.637, "read": 4.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.383, "count": 139, "max_ms": 31.743, "p95_ms": 25.22}, "gstreamer_push": {"avg_ms": 1.226, "count": 139, "max_ms": 1.498, "p95_ms": 1.333}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.33, "count": 139, "max_ms": 34.964, "p95_ms": 33.527}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.464, "count": 139, "max_ms": 49.224, "p95_ms": 48.3}, "yolo11_640_fire_11class": {"avg_ms": 43.732, "count": 139, "max_ms": 48.069, "p95_ms": 47.248}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.145, "count": 139, "max_ms": 49.767, "p95_ms": 48.855}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.483, "count": 139, "max_ms": 85.154, "p95_ms": 81.12}, "read_interval": {"avg_ms": 217.074, "count": 138, "max_ms": 10119.573, "p95_ms": 97.265}, "triton_wait": {"avg_ms": 45.219, "count": 139, "max_ms": 49.79, "p95_ms": 48.865}, "visualize": {"avg_ms": 3.361, "count": 139, "max_ms": 6.776, "p95_ms": 3.643}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10892}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10647.379, "limit_mb": 8796093022207.996}, "detail": {"pid": 78094, "restart_count": 411, "start_count": 412}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 377, "h264_decode_error_count": 1239, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:27:39+09:00", "last_error_summary": "[h264 @ 0x563d65894080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d65894080] error while decoding MB 0 9, bytestream 5725 [h264 @ 0x563d661163c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563d661163c0] error while decoding MB 0 35, bytestream 3571 [h264 @ 0x563d6596ad00] left block unavailable for requested intra mode [h264 @ 0x563d6596ad00] error while decoding MB 0 13, bytestream 5530 [in#0/rtsp @ 0x563d65824e00] Error during demuxing: Connection ", "last_exit_at": "2026-07-01T04:27:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:27:42+09:00", "pid": 78094, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.41, "name": "ffmpeg", "pid": 78094, "rss_mb": 138.41, "state": "S (sleeping)", "threads": 106}, "restart_count": 411, "restart_count_per_hour": 8, "rtsp_timeout_count": 370, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 412}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.909, "sample_count": 2691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1307153, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:27:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.459, "last_frame_pushed_at": "2026-07-01T04:27:39+09:00", "last_frame_read_age_sec": 4.543, "last_frame_read_at": "2026-07-01T04:27:39+09:00", "mjpeg": {"age_sec": 4.444, "exists": true, "mtime": "2026-07-01T04:27:39.834001+09:00", "mtime_epoch": 1782847659.8340013, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127580}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:27:44+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4988.551, "name": "celery", "pid": 267, "rss_mb": 4988.551, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 89, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.007, "pushed": 3.007, "read": 2.974}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.872, "count": 90, "max_ms": 31.743, "p95_ms": 24.993}, "gstreamer_push": {"avg_ms": 1.214, "count": 90, "max_ms": 1.366, "p95_ms": 1.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.232, "count": 90, "max_ms": 34.964, "p95_ms": 33.527}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 44.407, "count": 90, "max_ms": 49.224, "p95_ms": 48.525}, "yolo11_640_fire_11class": {"avg_ms": 43.706, "count": 90, "max_ms": 48.069, "p95_ms": 47.627}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 45.088, "count": 90, "max_ms": 49.767, "p95_ms": 49.038}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.854, "count": 90, "max_ms": 85.154, "p95_ms": 82.063}, "read_interval": {"avg_ms": 286.078, "count": 89, "max_ms": 10119.573, "p95_ms": 97.392}, "triton_wait": {"avg_ms": 45.19, "count": 90, "max_ms": 49.79, "p95_ms": 49.047}, "visualize": {"avg_ms": 3.355, "count": 90, "max_ms": 6.776, "p95_ms": 3.699}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12027}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10477.621, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x557199324200] missing picture in access unit with size 18 [h264 @ 0x5571998b0140] no frame! [in#0/rtsp @ 0x55719931fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55719931fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557199324840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 185, "h264_decode_error_count": 190, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:27:52+09:00", "last_error_summary": "[NULL @ 0x557199324200] missing picture in access unit with size 18 [h264 @ 0x5571998b0140] no frame! [in#0/rtsp @ 0x55719931fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55719931fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557199324840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T04:19:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 156.929, "last_started_at": "2026-07-01T04:25:15+09:00", "pid": 77942, "process": {"exists": false, "pid": 77942}, "restart_count": 184, "restart_count_per_hour": 9, "rtsp_timeout_count": 300, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 185}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.224, "sample_count": 2195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1443335, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:27:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T04:27:52+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T04:27:52+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T04:27:52.030171+09:00", "mtime_epoch": 1782847672.0301714, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94091}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:27:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.215, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.699, "pushed": 0.699, "read": 0.757}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.735, "count": 12, "max_ms": 20.587, "p95_ms": 20.587}, "gstreamer_push": {"avg_ms": 1.267, "count": 12, "max_ms": 2.033, "p95_ms": 2.033}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.385, "count": 12, "max_ms": 56.519, "p95_ms": 56.519}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.235, "count": 12, "max_ms": 66.054, "p95_ms": 66.054}, "yolo11_640_fire_11class": {"avg_ms": 56.051, "count": 12, "max_ms": 64.133, "p95_ms": 64.133}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.372, "count": 12, "max_ms": 73.577, "p95_ms": 73.577}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.625, "count": 12, "max_ms": 100.172, "p95_ms": 100.172}, "read_interval": {"avg_ms": 2720.799, "count": 13, "max_ms": 18258.688, "p95_ms": 18258.688}, "triton_wait": {"avg_ms": 60.458, "count": 12, "max_ms": 73.588, "p95_ms": 73.588}, "visualize": {"avg_ms": 3.623, "count": 12, "max_ms": 5.079, "p95_ms": 5.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.177, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12027}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10476.734, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:27:52+09:00", "last_error_summary": "[NULL @ 0x557199324200] missing picture in access unit with size 18 [h264 @ 0x5571998b0140] no frame! [in#0/rtsp @ 0x55719931fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55719931fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557199324840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 185, "h264_decode_error_count": 190, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:27:52+09:00", "last_error_summary": "[NULL @ 0x557199324200] missing picture in access unit with size 18 [h264 @ 0x5571998b0140] no frame! [in#0/rtsp @ 0x55719931fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55719931fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557199324840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T04:27:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 156.976, "last_started_at": "2026-07-01T04:25:15+09:00", "pid": 77942, "process": {"exists": false, "pid": 77942}, "restart_count": 184, "restart_count_per_hour": 9, "rtsp_timeout_count": 300, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 185}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.228, "sample_count": 2195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1443336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:27:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T04:27:52+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-01T04:27:52+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T04:27:52.030171+09:00", "mtime_epoch": 1782847672.0301714, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94091}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:27:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.215, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.755, "pushed": 0.755, "read": 0.755}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.897, "count": 13, "max_ms": 20.842, "p95_ms": 20.842}, "gstreamer_push": {"avg_ms": 1.265, "count": 13, "max_ms": 2.033, "p95_ms": 2.033}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.701, "count": 13, "max_ms": 56.519, "p95_ms": 56.519}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.087, "count": 13, "max_ms": 68.305, "p95_ms": 68.305}, "yolo11_640_fire_11class": {"avg_ms": 56.992, "count": 13, "max_ms": 68.292, "p95_ms": 68.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.501, "count": 13, "max_ms": 75.047, "p95_ms": 75.047}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.891, "count": 13, "max_ms": 104.091, "p95_ms": 104.091}, "read_interval": {"avg_ms": 2720.799, "count": 13, "max_ms": 18258.688, "p95_ms": 18258.688}, "triton_wait": {"avg_ms": 61.581, "count": 13, "max_ms": 75.059, "p95_ms": 75.059}, "visualize": {"avg_ms": 3.616, "count": 13, "max_ms": 5.079, "p95_ms": 5.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.223, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12027}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10476.684, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 185, "h264_decode_error_count": 190, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:27:52+09:00", "last_error_summary": "[NULL @ 0x557199324200] missing picture in access unit with size 18 [h264 @ 0x5571998b0140] no frame! [in#0/rtsp @ 0x55719931fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55719931fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557199324840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T04:27:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 156.982, "last_started_at": "2026-07-01T04:25:15+09:00", "pid": 77942, "process": {"exists": false, "pid": 77942}, "restart_count": 184, "restart_count_per_hour": 9, "rtsp_timeout_count": 300, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 185}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.228, "sample_count": 2195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1443336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:27:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T04:27:52+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-01T04:27:52+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-01T04:27:52.030171+09:00", "mtime_epoch": 1782847672.0301714, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94091}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:27:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.215, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.755, "pushed": 0.755, "read": 0.755}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.897, "count": 13, "max_ms": 20.842, "p95_ms": 20.842}, "gstreamer_push": {"avg_ms": 1.265, "count": 13, "max_ms": 2.033, "p95_ms": 2.033}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.701, "count": 13, "max_ms": 56.519, "p95_ms": 56.519}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.087, "count": 13, "max_ms": 68.305, "p95_ms": 68.305}, "yolo11_640_fire_11class": {"avg_ms": 56.992, "count": 13, "max_ms": 68.292, "p95_ms": 68.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.501, "count": 13, "max_ms": 75.047, "p95_ms": 75.047}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.891, "count": 13, "max_ms": 104.091, "p95_ms": 104.091}, "read_interval": {"avg_ms": 2720.799, "count": 13, "max_ms": 18258.688, "p95_ms": 18258.688}, "triton_wait": {"avg_ms": 61.581, "count": 13, "max_ms": 75.059, "p95_ms": 75.059}, "visualize": {"avg_ms": 3.616, "count": 13, "max_ms": 5.079, "p95_ms": 5.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.229, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12027}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10652.051, "limit_mb": 8796093022207.996}, "detail": {"pid": 78201, "restart_count": 185, "start_count": 186}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 185, "h264_decode_error_count": 190, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:27:52+09:00", "last_error_summary": "[NULL @ 0x557199324200] missing picture in access unit with size 18 [h264 @ 0x5571998b0140] no frame! [in#0/rtsp @ 0x55719931fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55719931fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x557199324840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T04:27:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:27:54+09:00", "pid": 78201, "process": {"exists": true, "fd_count": 5, "hwm_mb": 203.398, "name": "ffmpeg", "pid": 78201, "rss_mb": 203.398, "state": "S (sleeping)", "threads": 106}, "restart_count": 185, "restart_count_per_hour": 10, "rtsp_timeout_count": 300, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 186}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.228, "sample_count": 2158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1443336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:27:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.25, "last_frame_pushed_at": "2026-07-01T04:27:52+09:00", "last_frame_read_age_sec": 4.356, "last_frame_read_at": "2026-07-01T04:27:52+09:00", "mjpeg": {"age_sec": 4.234, "exists": true, "mtime": "2026-07-01T04:27:52.150173+09:00", "mtime_epoch": 1782847672.1501732, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95129}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:27:56+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.215, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.606, "pushed": 0.606, "read": 0.606}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.897, "count": 13, "max_ms": 20.842, "p95_ms": 20.842}, "gstreamer_push": {"avg_ms": 1.265, "count": 13, "max_ms": 2.033, "p95_ms": 2.033}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.701, "count": 13, "max_ms": 56.519, "p95_ms": 56.519}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.087, "count": 13, "max_ms": 68.305, "p95_ms": 68.305}, "yolo11_640_fire_11class": {"avg_ms": 56.992, "count": 13, "max_ms": 68.292, "p95_ms": 68.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.501, "count": 13, "max_ms": 75.047, "p95_ms": 75.047}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.891, "count": 13, "max_ms": 104.091, "p95_ms": 104.091}, "read_interval": {"avg_ms": 2720.799, "count": 13, "max_ms": 18258.688, "p95_ms": 18258.688}, "triton_wait": {"avg_ms": 61.581, "count": 13, "max_ms": 75.059, "p95_ms": 75.059}, "visualize": {"avg_ms": 3.616, "count": 13, "max_ms": 5.079, "p95_ms": 5.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.468, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10488.039, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55715069b180] error while decoding MB 113 18, bytestream -5 [in#0/rtsp @ 0x557150613e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557150613e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 186, "h264_decode_error_count": 191, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:29:55+09:00", "last_error_summary": "[h264 @ 0x55715069b180] error while decoding MB 113 18, bytestream -5 [in#0/rtsp @ 0x557150613e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557150613e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:27:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.444, "last_started_at": "2026-07-01T04:27:54+09:00", "pid": 78201, "process": {"exists": false, "pid": 78201}, "restart_count": 185, "restart_count_per_hour": 10, "rtsp_timeout_count": 302, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 186}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.071, "sample_count": 1838, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:29:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-01T04:29:55+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T04:29:55+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T04:29:55.767921+09:00", "mtime_epoch": 1782847795.7679214, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96856}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:29:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.473, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.686, "pushed": 0.686, "read": 0.743}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.763, "count": 12, "max_ms": 23.572, "p95_ms": 23.572}, "gstreamer_push": {"avg_ms": 1.229, "count": 12, "max_ms": 1.306, "p95_ms": 1.306}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.752, "count": 12, "max_ms": 50.108, "p95_ms": 50.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.471, "count": 12, "max_ms": 49.333, "p95_ms": 49.333}, "yolo11_640_fire_11class": {"avg_ms": 46.976, "count": 12, "max_ms": 48.354, "p95_ms": 48.354}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.777, "count": 12, "max_ms": 51.381, "p95_ms": 51.381}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.896, "count": 12, "max_ms": 80.831, "p95_ms": 80.831}, "read_interval": {"avg_ms": 2694.023, "count": 13, "max_ms": 17598.725, "p95_ms": 17598.725}, "triton_wait": {"avg_ms": 48.954, "count": 12, "max_ms": 51.393, "p95_ms": 51.393}, "visualize": {"avg_ms": 3.459, "count": 12, "max_ms": 4.017, "p95_ms": 4.017}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.492, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10488.027, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:29:55+09:00", "last_error_summary": "[h264 @ 0x55715069b180] error while decoding MB 113 18, bytestream -5 [in#0/rtsp @ 0x557150613e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557150613e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 186, "h264_decode_error_count": 191, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:29:55+09:00", "last_error_summary": "[h264 @ 0x55715069b180] error while decoding MB 113 18, bytestream -5 [in#0/rtsp @ 0x557150613e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557150613e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:29:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.469, "last_started_at": "2026-07-01T04:27:54+09:00", "pid": 78201, "process": {"exists": false, "pid": 78201}, "restart_count": 185, "restart_count_per_hour": 10, "rtsp_timeout_count": 302, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 186}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.071, "sample_count": 1839, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444082, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:29:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T04:29:55+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-01T04:29:55+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-01T04:29:55.767921+09:00", "mtime_epoch": 1782847795.7679214, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96856}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:29:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.473, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.742, "pushed": 0.742, "read": 0.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.86, "count": 13, "max_ms": 23.572, "p95_ms": 23.572}, "gstreamer_push": {"avg_ms": 1.243, "count": 13, "max_ms": 1.414, "p95_ms": 1.414}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.533, "count": 13, "max_ms": 50.108, "p95_ms": 50.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.558, "count": 13, "max_ms": 49.333, "p95_ms": 49.333}, "yolo11_640_fire_11class": {"avg_ms": 47.052, "count": 13, "max_ms": 48.354, "p95_ms": 48.354}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.801, "count": 13, "max_ms": 51.381, "p95_ms": 51.381}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.231, "count": 13, "max_ms": 83.249, "p95_ms": 83.249}, "read_interval": {"avg_ms": 2694.023, "count": 13, "max_ms": 17598.725, "p95_ms": 17598.725}, "triton_wait": {"avg_ms": 48.965, "count": 13, "max_ms": 51.393, "p95_ms": 51.393}, "visualize": {"avg_ms": 3.6, "count": 13, "max_ms": 5.297, "p95_ms": 5.297}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.517, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10488.145, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 186, "h264_decode_error_count": 191, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:29:55+09:00", "last_error_summary": "[h264 @ 0x55715069b180] error while decoding MB 113 18, bytestream -5 [in#0/rtsp @ 0x557150613e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557150613e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:29:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.476, "last_started_at": "2026-07-01T04:27:54+09:00", "pid": 78201, "process": {"exists": false, "pid": 78201}, "restart_count": 185, "restart_count_per_hour": 10, "rtsp_timeout_count": 302, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 186}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.071, "sample_count": 1839, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444082, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:29:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T04:29:55+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T04:29:55+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T04:29:55.767921+09:00", "mtime_epoch": 1782847795.7679214, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96856}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:29:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.473, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.742, "pushed": 0.742, "read": 0.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.86, "count": 13, "max_ms": 23.572, "p95_ms": 23.572}, "gstreamer_push": {"avg_ms": 1.243, "count": 13, "max_ms": 1.414, "p95_ms": 1.414}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.533, "count": 13, "max_ms": 50.108, "p95_ms": 50.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.558, "count": 13, "max_ms": 49.333, "p95_ms": 49.333}, "yolo11_640_fire_11class": {"avg_ms": 47.052, "count": 13, "max_ms": 48.354, "p95_ms": 48.354}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.801, "count": 13, "max_ms": 51.381, "p95_ms": 51.381}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.231, "count": 13, "max_ms": 83.249, "p95_ms": 83.249}, "read_interval": {"avg_ms": 2694.023, "count": 13, "max_ms": 17598.725, "p95_ms": 17598.725}, "triton_wait": {"avg_ms": 48.965, "count": 13, "max_ms": 51.393, "p95_ms": 51.393}, "visualize": {"avg_ms": 3.6, "count": 13, "max_ms": 5.297, "p95_ms": 5.297}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.523, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10611.301, "limit_mb": 8796093022207.996}, "detail": {"pid": 78353, "restart_count": 186, "start_count": 187}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 186, "h264_decode_error_count": 191, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:29:55+09:00", "last_error_summary": "[h264 @ 0x55715069b180] error while decoding MB 113 18, bytestream -5 [in#0/rtsp @ 0x557150613e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557150613e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:29:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:29:58+09:00", "pid": 78353, "process": {"exists": true, "fd_count": 5, "hwm_mb": 155.902, "name": "ffmpeg", "pid": 78353, "rss_mb": 155.902, "state": "R (running)", "threads": 49}, "restart_count": 186, "restart_count_per_hour": 11, "rtsp_timeout_count": 302, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 187}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.071, "sample_count": 1839, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444082, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:29:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.32, "last_frame_pushed_at": "2026-07-01T04:29:55+09:00", "last_frame_read_age_sec": 4.406, "last_frame_read_at": "2026-07-01T04:29:55+09:00", "mjpeg": {"age_sec": 4.305, "exists": true, "mtime": "2026-07-01T04:29:55.859923+09:00", "mtime_epoch": 1782847795.859923, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97404}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:30:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.473, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.596, "pushed": 0.596, "read": 0.596}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.86, "count": 13, "max_ms": 23.572, "p95_ms": 23.572}, "gstreamer_push": {"avg_ms": 1.243, "count": 13, "max_ms": 1.414, "p95_ms": 1.414}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.533, "count": 13, "max_ms": 50.108, "p95_ms": 50.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.558, "count": 13, "max_ms": 49.333, "p95_ms": 49.333}, "yolo11_640_fire_11class": {"avg_ms": 47.052, "count": 13, "max_ms": 48.354, "p95_ms": 48.354}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.801, "count": 13, "max_ms": 51.381, "p95_ms": 51.381}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.231, "count": 13, "max_ms": 83.249, "p95_ms": 83.249}, "read_interval": {"avg_ms": 2694.023, "count": 13, "max_ms": 17598.725, "p95_ms": 17598.725}, "triton_wait": {"avg_ms": 48.965, "count": 13, "max_ms": 51.393, "p95_ms": 51.393}, "visualize": {"avg_ms": 3.6, "count": 13, "max_ms": 5.297, "p95_ms": 5.297}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.83, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12038}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10502.527, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562e0a41bc00] error while decoding MB 101 12, bytestream -5 [in#0/rtsp @ 0x562e09a7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562e09a7ee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 187, "h264_decode_error_count": 192, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:31:35+09:00", "last_error_summary": "[h264 @ 0x562e0a41bc00] error while decoding MB 101 12, bytestream -5 [in#0/rtsp @ 0x562e09a7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562e09a7ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:29:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 96.851, "last_started_at": "2026-07-01T04:29:58+09:00", "pid": 78353, "process": {"exists": false, "pid": 78353}, "restart_count": 186, "restart_count_per_hour": 11, "rtsp_timeout_count": 304, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 187}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.189, "sample_count": 1675, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444610, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:31:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-01T04:31:34+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T04:31:34+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T04:31:34.953351+09:00", "mtime_epoch": 1782847894.9533505, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105238}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:31:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.73, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.801}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.606, "count": 13, "max_ms": 23.945, "p95_ms": 23.945}, "gstreamer_push": {"avg_ms": 1.216, "count": 13, "max_ms": 1.283, "p95_ms": 1.283}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.699, "count": 13, "max_ms": 58.862, "p95_ms": 58.862}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.931, "count": 13, "max_ms": 59.96, "p95_ms": 59.96}, "yolo11_640_fire_11class": {"avg_ms": 48.601, "count": 13, "max_ms": 59.455, "p95_ms": 59.455}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.112, "count": 13, "max_ms": 67.717, "p95_ms": 67.717}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.859, "count": 13, "max_ms": 93.985, "p95_ms": 93.985}, "read_interval": {"avg_ms": 2672.555, "count": 14, "max_ms": 20009.045, "p95_ms": 20009.045}, "triton_wait": {"avg_ms": 50.123, "count": 13, "max_ms": 67.728, "p95_ms": 67.728}, "visualize": {"avg_ms": 3.447, "count": 13, "max_ms": 3.604, "p95_ms": 3.604}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.473, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12038}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10502.324, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:31:35+09:00", "last_error_summary": "[h264 @ 0x562e0a41bc00] error while decoding MB 101 12, bytestream -5 [in#0/rtsp @ 0x562e09a7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562e09a7ee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 187, "h264_decode_error_count": 192, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:31:35+09:00", "last_error_summary": "[h264 @ 0x562e0a41bc00] error while decoding MB 101 12, bytestream -5 [in#0/rtsp @ 0x562e09a7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562e09a7ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:31:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 96.875, "last_started_at": "2026-07-01T04:29:58+09:00", "pid": 78353, "process": {"exists": false, "pid": 78353}, "restart_count": 186, "restart_count_per_hour": 11, "rtsp_timeout_count": 304, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 187}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.189, "sample_count": 1676, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444611, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:31:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T04:31:35+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-01T04:31:34+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-01T04:31:34.953351+09:00", "mtime_epoch": 1782847894.9533505, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105238}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:31:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.73, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.8, "pushed": 0.8, "read": 0.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.72, "count": 14, "max_ms": 23.945, "p95_ms": 23.945}, "gstreamer_push": {"avg_ms": 1.225, "count": 14, "max_ms": 1.347, "p95_ms": 1.347}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.586, "count": 14, "max_ms": 58.862, "p95_ms": 58.862}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.763, "count": 14, "max_ms": 59.96, "p95_ms": 59.96}, "yolo11_640_fire_11class": {"avg_ms": 48.415, "count": 14, "max_ms": 59.455, "p95_ms": 59.455}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.914, "count": 14, "max_ms": 67.717, "p95_ms": 67.717}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.934, "count": 14, "max_ms": 93.985, "p95_ms": 93.985}, "read_interval": {"avg_ms": 2672.555, "count": 14, "max_ms": 20009.045, "p95_ms": 20009.045}, "triton_wait": {"avg_ms": 49.925, "count": 14, "max_ms": 67.728, "p95_ms": 67.728}, "visualize": {"avg_ms": 3.577, "count": 14, "max_ms": 5.274, "p95_ms": 5.274}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.497, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12038}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10502.289, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 187, "h264_decode_error_count": 192, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:31:35+09:00", "last_error_summary": "[h264 @ 0x562e0a41bc00] error while decoding MB 101 12, bytestream -5 [in#0/rtsp @ 0x562e09a7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562e09a7ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:31:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 96.882, "last_started_at": "2026-07-01T04:29:58+09:00", "pid": 78353, "process": {"exists": false, "pid": 78353}, "restart_count": 186, "restart_count_per_hour": 11, "rtsp_timeout_count": 304, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 187}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.189, "sample_count": 1676, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444611, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:31:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T04:31:35+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-01T04:31:34+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-01T04:31:34.953351+09:00", "mtime_epoch": 1782847894.9533505, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105238}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:31:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.73, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.8, "pushed": 0.8, "read": 0.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.72, "count": 14, "max_ms": 23.945, "p95_ms": 23.945}, "gstreamer_push": {"avg_ms": 1.225, "count": 14, "max_ms": 1.347, "p95_ms": 1.347}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.586, "count": 14, "max_ms": 58.862, "p95_ms": 58.862}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.763, "count": 14, "max_ms": 59.96, "p95_ms": 59.96}, "yolo11_640_fire_11class": {"avg_ms": 48.415, "count": 14, "max_ms": 59.455, "p95_ms": 59.455}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.914, "count": 14, "max_ms": 67.717, "p95_ms": 67.717}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.934, "count": 14, "max_ms": 93.985, "p95_ms": 93.985}, "read_interval": {"avg_ms": 2672.555, "count": 14, "max_ms": 20009.045, "p95_ms": 20009.045}, "triton_wait": {"avg_ms": 49.925, "count": 14, "max_ms": 67.728, "p95_ms": 67.728}, "visualize": {"avg_ms": 3.577, "count": 14, "max_ms": 5.274, "p95_ms": 5.274}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.504, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12038}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10662.785, "limit_mb": 8796093022207.996}, "detail": {"pid": 78505, "restart_count": 187, "start_count": 188}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 187, "h264_decode_error_count": 192, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:31:35+09:00", "last_error_summary": "[h264 @ 0x562e0a41bc00] error while decoding MB 101 12, bytestream -5 [in#0/rtsp @ 0x562e09a7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562e09a7ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:31:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:31:37+09:00", "pid": 78505, "process": {"exists": true, "fd_count": 5, "hwm_mb": 189.543, "name": "ffmpeg", "pid": 78505, "rss_mb": 189.543, "state": "S (sleeping)", "threads": 106}, "restart_count": 187, "restart_count_per_hour": 12, "rtsp_timeout_count": 304, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 188}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.214, "sample_count": 1639, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444611, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:31:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.223, "last_frame_pushed_at": "2026-07-01T04:31:35+09:00", "last_frame_read_age_sec": 4.306, "last_frame_read_at": "2026-07-01T04:31:34+09:00", "mjpeg": {"age_sec": 4.21, "exists": true, "mtime": "2026-07-01T04:31:35.045352+09:00", "mtime_epoch": 1782847895.0453517, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103650}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:31:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1742.73, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.645, "pushed": 0.645, "read": 0.645}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.72, "count": 14, "max_ms": 23.945, "p95_ms": 23.945}, "gstreamer_push": {"avg_ms": 1.225, "count": 14, "max_ms": 1.347, "p95_ms": 1.347}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.586, "count": 14, "max_ms": 58.862, "p95_ms": 58.862}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.763, "count": 14, "max_ms": 59.96, "p95_ms": 59.96}, "yolo11_640_fire_11class": {"avg_ms": 48.415, "count": 14, "max_ms": 59.455, "p95_ms": 59.455}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.914, "count": 14, "max_ms": 67.717, "p95_ms": 67.717}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.934, "count": 14, "max_ms": 93.985, "p95_ms": 93.985}, "read_interval": {"avg_ms": 2672.555, "count": 14, "max_ms": 20009.045, "p95_ms": 20009.045}, "triton_wait": {"avg_ms": 49.925, "count": 14, "max_ms": 67.728, "p95_ms": 67.728}, "visualize": {"avg_ms": 3.577, "count": 14, "max_ms": 5.274, "p95_ms": 5.274}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.713, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10506.52, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c48333cdc0] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x55c482b34e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c482b34e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 188, "h264_decode_error_count": 193, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:34:00+09:00", "last_error_summary": "[h264 @ 0x55c48333cdc0] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x55c482b34e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c482b34e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:31:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 143.003, "last_started_at": "2026-07-01T04:31:37+09:00", "pid": 78505, "process": {"exists": false, "pid": 78505}, "restart_count": 187, "restart_count_per_hour": 11, "rtsp_timeout_count": 306, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 188}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.224, "sample_count": 1706, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1445596, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:34:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.083, "last_frame_pushed_at": "2026-07-01T04:34:00+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T04:34:00+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-01T04:34:00.191476+09:00", "mtime_epoch": 1782848040.1914759, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101158}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:34:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1743.246, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.734, "pushed": 0.734, "read": 0.791}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.193, "count": 13, "max_ms": 23.736, "p95_ms": 23.736}, "gstreamer_push": {"avg_ms": 1.237, "count": 13, "max_ms": 1.317, "p95_ms": 1.317}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.077, "count": 13, "max_ms": 50.133, "p95_ms": 50.133}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.307, "count": 13, "max_ms": 50.743, "p95_ms": 50.743}, "yolo11_640_fire_11class": {"avg_ms": 46.51, "count": 13, "max_ms": 49.301, "p95_ms": 49.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.117, "count": 13, "max_ms": 51.236, "p95_ms": 51.236}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.626, "count": 13, "max_ms": 82.353, "p95_ms": 82.353}, "read_interval": {"avg_ms": 2648.532, "count": 14, "max_ms": 19442.672, "p95_ms": 19442.672}, "triton_wait": {"avg_ms": 48.315, "count": 13, "max_ms": 51.245, "p95_ms": 51.245}, "visualize": {"avg_ms": 3.427, "count": 13, "max_ms": 4.145, "p95_ms": 4.145}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.708, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10506.262, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:34:00+09:00", "last_error_summary": "[h264 @ 0x55c48333cdc0] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x55c482b34e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c482b34e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 188, "h264_decode_error_count": 193, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:34:00+09:00", "last_error_summary": "[h264 @ 0x55c48333cdc0] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x55c482b34e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c482b34e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:34:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 143.027, "last_started_at": "2026-07-01T04:31:37+09:00", "pid": 78505, "process": {"exists": false, "pid": 78505}, "restart_count": 187, "restart_count_per_hour": 11, "rtsp_timeout_count": 306, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 188}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.224, "sample_count": 1707, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1445597, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:34:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T04:34:00+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-01T04:34:00+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-01T04:34:00.191476+09:00", "mtime_epoch": 1782848040.1914759, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101158}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:34:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1743.246, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.79, "pushed": 0.79, "read": 0.79}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.493, "count": 14, "max_ms": 26.397, "p95_ms": 26.397}, "gstreamer_push": {"avg_ms": 1.247, "count": 14, "max_ms": 1.382, "p95_ms": 1.382}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.304, "count": 14, "max_ms": 50.133, "p95_ms": 50.133}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.312, "count": 14, "max_ms": 50.743, "p95_ms": 50.743}, "yolo11_640_fire_11class": {"avg_ms": 46.571, "count": 14, "max_ms": 49.301, "p95_ms": 49.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.133, "count": 14, "max_ms": 51.236, "p95_ms": 51.236}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.289, "count": 14, "max_ms": 87.904, "p95_ms": 87.904}, "read_interval": {"avg_ms": 2648.532, "count": 14, "max_ms": 19442.672, "p95_ms": 19442.672}, "triton_wait": {"avg_ms": 48.318, "count": 14, "max_ms": 51.245, "p95_ms": 51.245}, "visualize": {"avg_ms": 3.621, "count": 14, "max_ms": 6.151, "p95_ms": 6.151}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.732, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10506.227, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 188, "h264_decode_error_count": 193, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:34:00+09:00", "last_error_summary": "[h264 @ 0x55c48333cdc0] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x55c482b34e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c482b34e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:34:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 143.033, "last_started_at": "2026-07-01T04:31:37+09:00", "pid": 78505, "process": {"exists": false, "pid": 78505}, "restart_count": 187, "restart_count_per_hour": 11, "rtsp_timeout_count": 306, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 188}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.224, "sample_count": 1706, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1445597, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:34:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T04:34:00+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-01T04:34:00+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-01T04:34:00.191476+09:00", "mtime_epoch": 1782848040.1914759, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101158}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:34:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1743.246, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.789, "pushed": 0.789, "read": 0.789}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.493, "count": 14, "max_ms": 26.397, "p95_ms": 26.397}, "gstreamer_push": {"avg_ms": 1.247, "count": 14, "max_ms": 1.382, "p95_ms": 1.382}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.304, "count": 14, "max_ms": 50.133, "p95_ms": 50.133}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.312, "count": 14, "max_ms": 50.743, "p95_ms": 50.743}, "yolo11_640_fire_11class": {"avg_ms": 46.571, "count": 14, "max_ms": 49.301, "p95_ms": 49.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.133, "count": 14, "max_ms": 51.236, "p95_ms": 51.236}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.289, "count": 14, "max_ms": 87.904, "p95_ms": 87.904}, "read_interval": {"avg_ms": 2648.532, "count": 14, "max_ms": 19442.672, "p95_ms": 19442.672}, "triton_wait": {"avg_ms": 48.318, "count": 14, "max_ms": 51.245, "p95_ms": 51.245}, "visualize": {"avg_ms": 3.621, "count": 14, "max_ms": 6.151, "p95_ms": 6.151}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.738, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10526.762, "limit_mb": 8796093022207.996}, "detail": {"pid": 78657, "restart_count": 188, "start_count": 189}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 188, "h264_decode_error_count": 193, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:34:00+09:00", "last_error_summary": "[h264 @ 0x55c48333cdc0] error while decoding MB 116 31, bytestream -5 [in#0/rtsp @ 0x55c482b34e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c482b34e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:34:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:34:02+09:00", "pid": 78657, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.387, "name": "ffmpeg", "pid": 78657, "rss_mb": 57.387, "state": "S (sleeping)", "threads": 1}, "restart_count": 188, "restart_count_per_hour": 12, "rtsp_timeout_count": 306, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 189}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.305, "sample_count": 1668, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1445597, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:34:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.463, "last_frame_pushed_at": "2026-07-01T04:34:00+09:00", "last_frame_read_age_sec": 4.554, "last_frame_read_at": "2026-07-01T04:34:00+09:00", "mjpeg": {"age_sec": 4.445, "exists": true, "mtime": "2026-07-01T04:34:00.295478+09:00", "mtime_epoch": 1782848040.2954776, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100409}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:34:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1743.246, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.631, "pushed": 0.631, "read": 0.631}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.493, "count": 14, "max_ms": 26.397, "p95_ms": 26.397}, "gstreamer_push": {"avg_ms": 1.247, "count": 14, "max_ms": 1.382, "p95_ms": 1.382}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.304, "count": 14, "max_ms": 50.133, "p95_ms": 50.133}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.312, "count": 14, "max_ms": 50.743, "p95_ms": 50.743}, "yolo11_640_fire_11class": {"avg_ms": 46.571, "count": 14, "max_ms": 49.301, "p95_ms": 49.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.133, "count": 14, "max_ms": 51.236, "p95_ms": 51.236}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.289, "count": 14, "max_ms": 87.904, "p95_ms": 87.904}, "read_interval": {"avg_ms": 2648.532, "count": 14, "max_ms": 19442.672, "p95_ms": 19442.672}, "triton_wait": {"avg_ms": 48.318, "count": 14, "max_ms": 51.245, "p95_ms": 51.245}, "visualize": {"avg_ms": 3.621, "count": 14, "max_ms": 6.151, "p95_ms": 6.151}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.191, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12053}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10518.82, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5576dba88e00] error while decoding MB 109 38, bytestream -7 [in#0/rtsp @ 0x5576daf47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576daf47e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 189, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:36:10+09:00", "last_error_summary": "[h264 @ 0x5576dba88e00] error while decoding MB 109 38, bytestream -7 [in#0/rtsp @ 0x5576daf47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576daf47e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:34:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.116, "last_started_at": "2026-07-01T04:34:02+09:00", "pid": 78657, "process": {"exists": false, "pid": 78657}, "restart_count": 188, "restart_count_per_hour": 12, "rtsp_timeout_count": 308, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 189}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.617, "sample_count": 1817, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446414, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:36:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T04:36:10+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-01T04:36:10+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T04:36:10.793414+09:00", "mtime_epoch": 1782848170.7934136, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103215}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:36:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1743.504, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.747, "pushed": 0.747, "read": 0.805}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.93, "count": 13, "max_ms": 24.786, "p95_ms": 24.786}, "gstreamer_push": {"avg_ms": 1.217, "count": 13, "max_ms": 1.279, "p95_ms": 1.279}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.334, "count": 13, "max_ms": 41.63, "p95_ms": 41.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.142, "count": 13, "max_ms": 49.356, "p95_ms": 49.356}, "yolo11_640_fire_11class": {"avg_ms": 45.844, "count": 13, "max_ms": 47.235, "p95_ms": 47.235}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.604, "count": 13, "max_ms": 50.901, "p95_ms": 50.901}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.778, "count": 13, "max_ms": 82.395, "p95_ms": 82.395}, "read_interval": {"avg_ms": 2556.565, "count": 14, "max_ms": 18468.245, "p95_ms": 18468.245}, "triton_wait": {"avg_ms": 48.985, "count": 13, "max_ms": 50.913, "p95_ms": 50.913}, "visualize": {"avg_ms": 3.338, "count": 13, "max_ms": 3.429, "p95_ms": 3.429}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.394, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12053}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10518.637, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:36:10+09:00", "last_error_summary": "[h264 @ 0x5576dba88e00] error while decoding MB 109 38, bytestream -7 [in#0/rtsp @ 0x5576daf47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576daf47e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 189, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:36:10+09:00", "last_error_summary": "[h264 @ 0x5576dba88e00] error while decoding MB 109 38, bytestream -7 [in#0/rtsp @ 0x5576daf47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576daf47e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:36:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.141, "last_started_at": "2026-07-01T04:34:02+09:00", "pid": 78657, "process": {"exists": false, "pid": 78657}, "restart_count": 188, "restart_count_per_hour": 12, "rtsp_timeout_count": 308, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 189}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.617, "sample_count": 1818, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446415, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:36:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T04:36:10+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-01T04:36:10+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-01T04:36:10.793414+09:00", "mtime_epoch": 1782848170.7934136, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103215}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:36:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1743.504, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.804, "pushed": 0.804, "read": 0.804}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.098, "count": 14, "max_ms": 24.786, "p95_ms": 24.786}, "gstreamer_push": {"avg_ms": 1.232, "count": 14, "max_ms": 1.433, "p95_ms": 1.433}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.482, "count": 14, "max_ms": 41.63, "p95_ms": 41.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.393, "count": 14, "max_ms": 50.656, "p95_ms": 50.656}, "yolo11_640_fire_11class": {"avg_ms": 45.95, "count": 14, "max_ms": 47.336, "p95_ms": 47.336}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.64, "count": 14, "max_ms": 50.901, "p95_ms": 50.901}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.29, "count": 14, "max_ms": 84.948, "p95_ms": 84.948}, "read_interval": {"avg_ms": 2556.565, "count": 14, "max_ms": 18468.245, "p95_ms": 18468.245}, "triton_wait": {"avg_ms": 49.105, "count": 14, "max_ms": 50.913, "p95_ms": 50.913}, "visualize": {"avg_ms": 3.528, "count": 14, "max_ms": 6.003, "p95_ms": 6.003}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.418, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12053}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10518.59, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 189, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:36:10+09:00", "last_error_summary": "[h264 @ 0x5576dba88e00] error while decoding MB 109 38, bytestream -7 [in#0/rtsp @ 0x5576daf47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576daf47e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:36:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.148, "last_started_at": "2026-07-01T04:34:02+09:00", "pid": 78657, "process": {"exists": false, "pid": 78657}, "restart_count": 188, "restart_count_per_hour": 12, "rtsp_timeout_count": 308, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 189}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.617, "sample_count": 1818, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446415, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:36:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T04:36:10+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-01T04:36:10+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-01T04:36:10.793414+09:00", "mtime_epoch": 1782848170.7934136, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103215}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:36:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1743.504, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.803, "pushed": 0.803, "read": 0.803}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.098, "count": 14, "max_ms": 24.786, "p95_ms": 24.786}, "gstreamer_push": {"avg_ms": 1.232, "count": 14, "max_ms": 1.433, "p95_ms": 1.433}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.482, "count": 14, "max_ms": 41.63, "p95_ms": 41.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.393, "count": 14, "max_ms": 50.656, "p95_ms": 50.656}, "yolo11_640_fire_11class": {"avg_ms": 45.95, "count": 14, "max_ms": 47.336, "p95_ms": 47.336}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.64, "count": 14, "max_ms": 50.901, "p95_ms": 50.901}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.29, "count": 14, "max_ms": 84.948, "p95_ms": 84.948}, "read_interval": {"avg_ms": 2556.565, "count": 14, "max_ms": 18468.245, "p95_ms": 18468.245}, "triton_wait": {"avg_ms": 49.105, "count": 14, "max_ms": 50.913, "p95_ms": 50.913}, "visualize": {"avg_ms": 3.528, "count": 14, "max_ms": 6.003, "p95_ms": 6.003}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.425, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12053}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10539.801, "limit_mb": 8796093022207.996}, "detail": {"pid": 78809, "restart_count": 189, "start_count": 190}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 189, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:36:10+09:00", "last_error_summary": "[h264 @ 0x5576dba88e00] error while decoding MB 109 38, bytestream -7 [in#0/rtsp @ 0x5576daf47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5576daf47e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:36:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:36:13+09:00", "pid": 78809, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.031, "name": "ffmpeg", "pid": 78809, "rss_mb": 58.031, "state": "S (sleeping)", "threads": 1}, "restart_count": 189, "restart_count_per_hour": 13, "rtsp_timeout_count": 308, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 190}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.617, "sample_count": 1818, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446415, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:36:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.753, "last_frame_pushed_at": "2026-07-01T04:36:10+09:00", "last_frame_read_age_sec": 4.84, "last_frame_read_at": "2026-07-01T04:36:10+09:00", "mjpeg": {"age_sec": 4.737, "exists": true, "mtime": "2026-07-01T04:36:10.889415+09:00", "mtime_epoch": 1782848170.889415, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103059}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:36:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1743.504, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.632, "pushed": 0.632, "read": 0.632}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.098, "count": 14, "max_ms": 24.786, "p95_ms": 24.786}, "gstreamer_push": {"avg_ms": 1.232, "count": 14, "max_ms": 1.433, "p95_ms": 1.433}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.482, "count": 14, "max_ms": 41.63, "p95_ms": 41.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.393, "count": 14, "max_ms": 50.656, "p95_ms": 50.656}, "yolo11_640_fire_11class": {"avg_ms": 45.95, "count": 14, "max_ms": 47.336, "p95_ms": 47.336}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.64, "count": 14, "max_ms": 50.901, "p95_ms": 50.901}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.29, "count": 14, "max_ms": 84.948, "p95_ms": 84.948}, "read_interval": {"avg_ms": 2556.565, "count": 14, "max_ms": 18468.245, "p95_ms": 18468.245}, "triton_wait": {"avg_ms": 49.105, "count": 14, "max_ms": 50.913, "p95_ms": 50.913}, "visualize": {"avg_ms": 3.528, "count": 14, "max_ms": 6.003, "p95_ms": 6.003}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.164, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10935}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10546.809, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5644dbbfcb00] reference picture missing during reorder [h264 @ 0x5644dbbfcb00] Missing reference picture, default is 65547 [h264 @ 0x5644dbc10c00] mmco: unref short failure [h264 @ 0x5644dbc10c00] illegal short term buffer state detected [h264 @ 0x5644dc624f40] error while decoding MB 14 6, bytestream -5 [h264 @ 0x5644dc7c2c40] left block unavailable for requested intra mode [h264 @ 0x5644dc7c2c40] error while decoding MB 0 8, bytestream 49188 [h264 @ 0x5644dc5560c0] error while decodi", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 378, "h264_decode_error_count": 1242, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:36:44+09:00", "last_error_summary": "[h264 @ 0x5644dbbfcb00] reference picture missing during reorder [h264 @ 0x5644dbbfcb00] Missing reference picture, default is 65547 [h264 @ 0x5644dbc10c00] mmco: unref short failure [h264 @ 0x5644dbc10c00] illegal short term buffer state detected [h264 @ 0x5644dc624f40] error while decoding MB 14 6, bytestream -5 [h264 @ 0x5644dc7c2c40] left block unavailable for requested intra mode [h264 @ 0x5644dc7c2c40] error while decoding MB 0 8, bytestream 49188 [h264 @ 0x5644dc5560c0] error while decodi", "last_exit_at": "2026-07-01T04:27:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 542.314, "last_started_at": "2026-07-01T04:27:42+09:00", "pid": 78094, "process": {"exists": false, "pid": 78094}, "restart_count": 411, "restart_count_per_hour": 6, "rtsp_timeout_count": 372, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 412}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.498, "sample_count": 2750, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1312243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:36:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-01T04:36:44+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T04:36:44+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-01T04:36:44.541918+09:00", "mtime_epoch": 1782848204.5419178, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116636}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:36:44+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4990.871, "name": "celery", "pid": 267, "rss_mb": 4990.871, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 116, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.835, "pushed": 3.835, "read": 3.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.038, "count": 115, "max_ms": 35.666, "p95_ms": 23.994}, "gstreamer_push": {"avg_ms": 1.224, "count": 115, "max_ms": 1.329, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.753, "count": 115, "max_ms": 42.361, "p95_ms": 33.924}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 45.741, "count": 115, "max_ms": 63.621, "p95_ms": 61.161}, "yolo11_640_fire_11class": {"avg_ms": 45.044, "count": 115, "max_ms": 63.846, "p95_ms": 61.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 46.956, "count": 115, "max_ms": 76.54, "p95_ms": 70.887}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.13, "count": 115, "max_ms": 123.838, "p95_ms": 100.743}, "read_interval": {"avg_ms": 258.809, "count": 116, "max_ms": 10125.607, "p95_ms": 115.454}, "triton_wait": {"avg_ms": 47.092, "count": 115, "max_ms": 76.556, "p95_ms": 70.9}, "visualize": {"avg_ms": 3.404, "count": 115, "max_ms": 5.622, "p95_ms": 3.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10935}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10546.238, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:36:44+09:00", "last_error_summary": "[h264 @ 0x5644dbbfcb00] reference picture missing during reorder [h264 @ 0x5644dbbfcb00] Missing reference picture, default is 65547 [h264 @ 0x5644dbc10c00] mmco: unref short failure [h264 @ 0x5644dbc10c00] illegal short term buffer state detected [h264 @ 0x5644dc624f40] error while decoding MB 14 6, bytestream -5 [h264 @ 0x5644dc7c2c40] left block unavailable for requested intra mode [h264 @ 0x5644dc7c2c40] error while decoding MB 0 8, bytestream 49188 [h264 @ 0x5644dc5560c0] error while decodi", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 378, "h264_decode_error_count": 1242, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:36:44+09:00", "last_error_summary": "[h264 @ 0x5644dbbfcb00] reference picture missing during reorder [h264 @ 0x5644dbbfcb00] Missing reference picture, default is 65547 [h264 @ 0x5644dbc10c00] mmco: unref short failure [h264 @ 0x5644dbc10c00] illegal short term buffer state detected [h264 @ 0x5644dc624f40] error while decoding MB 14 6, bytestream -5 [h264 @ 0x5644dc7c2c40] left block unavailable for requested intra mode [h264 @ 0x5644dc7c2c40] error while decoding MB 0 8, bytestream 49188 [h264 @ 0x5644dc5560c0] error while decodi", "last_exit_at": "2026-07-01T04:36:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 542.375, "last_started_at": "2026-07-01T04:27:42+09:00", "pid": 78094, "process": {"exists": false, "pid": 78094}, "restart_count": 411, "restart_count_per_hour": 6, "rtsp_timeout_count": 372, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 412}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.498, "sample_count": 2750, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1312244, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:36:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T04:36:44+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-01T04:36:44+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-01T04:36:44.541918+09:00", "mtime_epoch": 1782848204.5419178, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116636}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:36:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4990.871, "name": "celery", "pid": 267, "rss_mb": 4990.871, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.87, "pushed": 3.87, "read": 3.836}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.033, "count": 116, "max_ms": 35.666, "p95_ms": 23.994}, "gstreamer_push": {"avg_ms": 1.224, "count": 116, "max_ms": 1.329, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.759, "count": 116, "max_ms": 42.361, "p95_ms": 33.924}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 45.926, "count": 116, "max_ms": 67.241, "p95_ms": 62.403}, "yolo11_640_fire_11class": {"avg_ms": 45.226, "count": 116, "max_ms": 66.083, "p95_ms": 61.273}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.203, "count": 116, "max_ms": 76.54, "p95_ms": 71.864}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.37, "count": 116, "max_ms": 123.838, "p95_ms": 101.281}, "read_interval": {"avg_ms": 260.296, "count": 115, "max_ms": 10125.607, "p95_ms": 115.454}, "triton_wait": {"avg_ms": 47.338, "count": 116, "max_ms": 76.556, "p95_ms": 71.878}, "visualize": {"avg_ms": 3.404, "count": 116, "max_ms": 5.622, "p95_ms": 3.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10935}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10546.379, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 378, "h264_decode_error_count": 1242, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:36:44+09:00", "last_error_summary": "[h264 @ 0x5644dbbfcb00] reference picture missing during reorder [h264 @ 0x5644dbbfcb00] Missing reference picture, default is 65547 [h264 @ 0x5644dbc10c00] mmco: unref short failure [h264 @ 0x5644dbc10c00] illegal short term buffer state detected [h264 @ 0x5644dc624f40] error while decoding MB 14 6, bytestream -5 [h264 @ 0x5644dc7c2c40] left block unavailable for requested intra mode [h264 @ 0x5644dc7c2c40] error while decoding MB 0 8, bytestream 49188 [h264 @ 0x5644dc5560c0] error while decodi", "last_exit_at": "2026-07-01T04:36:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 542.382, "last_started_at": "2026-07-01T04:27:42+09:00", "pid": 78094, "process": {"exists": false, "pid": 78094}, "restart_count": 411, "restart_count_per_hour": 6, "rtsp_timeout_count": 372, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 412}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.498, "sample_count": 2750, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1312244, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:36:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-01T04:36:44+09:00", "last_frame_read_age_sec": 0.123, "last_frame_read_at": "2026-07-01T04:36:44+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-01T04:36:44.541918+09:00", "mtime_epoch": 1782848204.5419178, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116636}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:36:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4990.871, "name": "celery", "pid": 267, "rss_mb": 4990.871, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.869, "pushed": 3.869, "read": 3.835}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.033, "count": 116, "max_ms": 35.666, "p95_ms": 23.994}, "gstreamer_push": {"avg_ms": 1.224, "count": 116, "max_ms": 1.329, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.759, "count": 116, "max_ms": 42.361, "p95_ms": 33.924}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 45.926, "count": 116, "max_ms": 67.241, "p95_ms": 62.403}, "yolo11_640_fire_11class": {"avg_ms": 45.226, "count": 116, "max_ms": 66.083, "p95_ms": 61.273}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.203, "count": 116, "max_ms": 76.54, "p95_ms": 71.864}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.37, "count": 116, "max_ms": 123.838, "p95_ms": 101.281}, "read_interval": {"avg_ms": 260.296, "count": 115, "max_ms": 10125.607, "p95_ms": 115.454}, "triton_wait": {"avg_ms": 47.338, "count": 116, "max_ms": 76.556, "p95_ms": 71.878}, "visualize": {"avg_ms": 3.404, "count": 116, "max_ms": 5.622, "p95_ms": 3.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10935}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10654.16, "limit_mb": 8796093022207.996}, "detail": {"pid": 78961, "restart_count": 412, "start_count": 413}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 378, "h264_decode_error_count": 1242, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:36:44+09:00", "last_error_summary": "[h264 @ 0x5644dbbfcb00] reference picture missing during reorder [h264 @ 0x5644dbbfcb00] Missing reference picture, default is 65547 [h264 @ 0x5644dbc10c00] mmco: unref short failure [h264 @ 0x5644dbc10c00] illegal short term buffer state detected [h264 @ 0x5644dc624f40] error while decoding MB 14 6, bytestream -5 [h264 @ 0x5644dc7c2c40] left block unavailable for requested intra mode [h264 @ 0x5644dc7c2c40] error while decoding MB 0 8, bytestream 49188 [h264 @ 0x5644dc5560c0] error while decodi", "last_exit_at": "2026-07-01T04:36:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:36:47+09:00", "pid": 78961, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.047, "name": "ffmpeg", "pid": 78961, "rss_mb": 139.047, "state": "S (sleeping)", "threads": 106}, "restart_count": 412, "restart_count_per_hour": 7, "rtsp_timeout_count": 372, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 413}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.528, "sample_count": 2703, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1312244, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:36:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.415, "last_frame_pushed_at": "2026-07-01T04:36:44+09:00", "last_frame_read_age_sec": 4.522, "last_frame_read_at": "2026-07-01T04:36:44+09:00", "mjpeg": {"age_sec": 4.4, "exists": true, "mtime": "2026-07-01T04:36:44.657919+09:00", "mtime_epoch": 1782848204.6579194, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117269}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:36:49+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4990.871, "name": "celery", "pid": 267, "rss_mb": 4990.871, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.235, "pushed": 2.235, "read": 2.202}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.541, "count": 67, "max_ms": 35.666, "p95_ms": 24.541}, "gstreamer_push": {"avg_ms": 1.226, "count": 67, "max_ms": 1.329, "p95_ms": 1.294}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.978, "count": 67, "max_ms": 42.361, "p95_ms": 39.466}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.922, "count": 67, "max_ms": 67.241, "p95_ms": 62.567}, "yolo11_640_fire_11class": {"avg_ms": 46.411, "count": 67, "max_ms": 66.083, "p95_ms": 61.811}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.724, "count": 67, "max_ms": 76.54, "p95_ms": 72.357}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.524, "count": 67, "max_ms": 123.838, "p95_ms": 102.492}, "read_interval": {"avg_ms": 386.759, "count": 66, "max_ms": 10125.607, "p95_ms": 120.73}, "triton_wait": {"avg_ms": 48.897, "count": 67, "max_ms": 76.556, "p95_ms": 72.369}, "visualize": {"avg_ms": 3.439, "count": 67, "max_ms": 5.622, "p95_ms": 3.552}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10497.551, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561d2ce5d3c0] error while decoding MB 40 29, bytestream -7 [in#0/rtsp @ 0x561d2c719e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d2c719e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 190, "h264_decode_error_count": 195, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:38:31+09:00", "last_error_summary": "[h264 @ 0x561d2ce5d3c0] error while decoding MB 40 29, bytestream -7 [in#0/rtsp @ 0x561d2c719e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d2c719e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:36:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.005, "last_started_at": "2026-07-01T04:36:13+09:00", "pid": 78809, "process": {"exists": false, "pid": 78809}, "restart_count": 189, "restart_count_per_hour": 13, "rtsp_timeout_count": 310, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 190}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.36, "sample_count": 1763, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1447346, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:38:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-01T04:38:31+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T04:38:31+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T04:38:31.567524+09:00", "mtime_epoch": 1782848311.5675237, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104789}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:38:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1744.051, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.74, "pushed": 0.74, "read": 0.797}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.233, "count": 13, "max_ms": 20.932, "p95_ms": 20.932}, "gstreamer_push": {"avg_ms": 1.231, "count": 13, "max_ms": 1.337, "p95_ms": 1.337}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.689, "count": 13, "max_ms": 47.995, "p95_ms": 47.995}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.781, "count": 13, "max_ms": 53.919, "p95_ms": 53.919}, "yolo11_640_fire_11class": {"avg_ms": 49.008, "count": 13, "max_ms": 52.839, "p95_ms": 52.839}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.691, "count": 13, "max_ms": 54.458, "p95_ms": 54.458}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.926, "count": 13, "max_ms": 83.437, "p95_ms": 83.437}, "read_interval": {"avg_ms": 2474.808, "count": 14, "max_ms": 17138.59, "p95_ms": 17138.59}, "triton_wait": {"avg_ms": 50.704, "count": 13, "max_ms": 54.468, "p95_ms": 54.468}, "visualize": {"avg_ms": 3.435, "count": 13, "max_ms": 3.964, "p95_ms": 3.964}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.577, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10497.348, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:38:31+09:00", "last_error_summary": "[h264 @ 0x561d2ce5d3c0] error while decoding MB 40 29, bytestream -7 [in#0/rtsp @ 0x561d2c719e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d2c719e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 190, "h264_decode_error_count": 195, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:38:31+09:00", "last_error_summary": "[h264 @ 0x561d2ce5d3c0] error while decoding MB 40 29, bytestream -7 [in#0/rtsp @ 0x561d2c719e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d2c719e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:38:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.028, "last_started_at": "2026-07-01T04:36:13+09:00", "pid": 78809, "process": {"exists": false, "pid": 78809}, "restart_count": 189, "restart_count_per_hour": 13, "rtsp_timeout_count": 310, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 190}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.36, "sample_count": 1764, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1447347, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:38:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T04:38:31+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-01T04:38:31+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-01T04:38:31.567524+09:00", "mtime_epoch": 1782848311.5675237, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104789}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:38:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1744.051, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.795, "pushed": 0.795, "read": 0.795}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.456, "count": 14, "max_ms": 23.364, "p95_ms": 23.364}, "gstreamer_push": {"avg_ms": 1.248, "count": 14, "max_ms": 1.458, "p95_ms": 1.458}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.719, "count": 14, "max_ms": 47.995, "p95_ms": 47.995}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.497, "count": 14, "max_ms": 53.919, "p95_ms": 53.919}, "yolo11_640_fire_11class": {"avg_ms": 48.742, "count": 14, "max_ms": 52.839, "p95_ms": 52.839}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.544, "count": 14, "max_ms": 54.458, "p95_ms": 54.458}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.249, "count": 14, "max_ms": 83.439, "p95_ms": 83.439}, "read_interval": {"avg_ms": 2474.808, "count": 14, "max_ms": 17138.59, "p95_ms": 17138.59}, "triton_wait": {"avg_ms": 50.557, "count": 14, "max_ms": 54.468, "p95_ms": 54.468}, "visualize": {"avg_ms": 3.643, "count": 14, "max_ms": 6.346, "p95_ms": 6.346}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.6, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10497.199, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 190, "h264_decode_error_count": 195, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:38:31+09:00", "last_error_summary": "[h264 @ 0x561d2ce5d3c0] error while decoding MB 40 29, bytestream -7 [in#0/rtsp @ 0x561d2c719e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d2c719e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:38:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.035, "last_started_at": "2026-07-01T04:36:13+09:00", "pid": 78809, "process": {"exists": false, "pid": 78809}, "restart_count": 189, "restart_count_per_hour": 13, "rtsp_timeout_count": 310, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 190}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.36, "sample_count": 1764, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1447347, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:38:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T04:38:31+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-01T04:38:31+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-01T04:38:31.567524+09:00", "mtime_epoch": 1782848311.5675237, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104789}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:38:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1744.051, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.795, "pushed": 0.795, "read": 0.795}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.456, "count": 14, "max_ms": 23.364, "p95_ms": 23.364}, "gstreamer_push": {"avg_ms": 1.248, "count": 14, "max_ms": 1.458, "p95_ms": 1.458}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.719, "count": 14, "max_ms": 47.995, "p95_ms": 47.995}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.497, "count": 14, "max_ms": 53.919, "p95_ms": 53.919}, "yolo11_640_fire_11class": {"avg_ms": 48.742, "count": 14, "max_ms": 52.839, "p95_ms": 52.839}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.544, "count": 14, "max_ms": 54.458, "p95_ms": 54.458}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.249, "count": 14, "max_ms": 83.439, "p95_ms": 83.439}, "read_interval": {"avg_ms": 2474.808, "count": 14, "max_ms": 17138.59, "p95_ms": 17138.59}, "triton_wait": {"avg_ms": 50.557, "count": 14, "max_ms": 54.468, "p95_ms": 54.468}, "visualize": {"avg_ms": 3.643, "count": 14, "max_ms": 6.346, "p95_ms": 6.346}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.607, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12061}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10610.453, "limit_mb": 8796093022207.996}, "detail": {"pid": 79068, "restart_count": 190, "start_count": 191}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 190, "h264_decode_error_count": 195, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:38:31+09:00", "last_error_summary": "[h264 @ 0x561d2ce5d3c0] error while decoding MB 40 29, bytestream -7 [in#0/rtsp @ 0x561d2c719e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d2c719e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:38:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:38:34+09:00", "pid": 79068, "process": {"exists": true, "fd_count": 5, "hwm_mb": 146.082, "name": "ffmpeg", "pid": 79068, "rss_mb": 146.082, "state": "R (running)", "threads": 49}, "restart_count": 190, "restart_count_per_hour": 14, "rtsp_timeout_count": 310, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 191}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.36, "sample_count": 1764, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1447347, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:38:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.415, "last_frame_pushed_at": "2026-07-01T04:38:31+09:00", "last_frame_read_age_sec": 4.501, "last_frame_read_at": "2026-07-01T04:38:31+09:00", "mjpeg": {"age_sec": 4.4, "exists": true, "mtime": "2026-07-01T04:38:31.663525+09:00", "mtime_epoch": 1782848311.663525, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104447}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:38:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1744.055, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.636, "pushed": 0.636, "read": 0.636}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.456, "count": 14, "max_ms": 23.364, "p95_ms": 23.364}, "gstreamer_push": {"avg_ms": 1.248, "count": 14, "max_ms": 1.458, "p95_ms": 1.458}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.719, "count": 14, "max_ms": 47.995, "p95_ms": 47.995}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.497, "count": 14, "max_ms": 53.919, "p95_ms": 53.919}, "yolo11_640_fire_11class": {"avg_ms": 48.742, "count": 14, "max_ms": 52.839, "p95_ms": 52.839}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.544, "count": 14, "max_ms": 54.458, "p95_ms": 54.458}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.249, "count": 14, "max_ms": 83.439, "p95_ms": 83.439}, "read_interval": {"avg_ms": 2474.808, "count": 14, "max_ms": 17138.59, "p95_ms": 17138.59}, "triton_wait": {"avg_ms": 50.557, "count": 14, "max_ms": 54.468, "p95_ms": 54.468}, "visualize": {"avg_ms": 3.643, "count": 14, "max_ms": 6.346, "p95_ms": 6.346}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.01, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10521.117, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b9e5647e80] error while decoding MB 9 4, bytestream -7 [in#0/rtsp @ 0x55b9e5604e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b9e5604e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 191, "h264_decode_error_count": 196, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:43:05+09:00", "last_error_summary": "[h264 @ 0x55b9e5647e80] error while decoding MB 9 4, bytestream -7 [in#0/rtsp @ 0x55b9e5604e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b9e5604e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:38:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 271.23, "last_started_at": "2026-07-01T04:38:34+09:00", "pid": 79068, "process": {"exists": false, "pid": 79068}, "restart_count": 190, "restart_count_per_hour": 12, "rtsp_timeout_count": 312, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 191}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.034, "sample_count": 2251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449584, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:43:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-01T04:43:05+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T04:43:05+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T04:43:05.231016+09:00", "mtime_epoch": 1782848585.231016, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107503}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:43:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1745.227, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.676, "pushed": 0.676, "read": 0.732}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.111, "count": 12, "max_ms": 22.401, "p95_ms": 22.401}, "gstreamer_push": {"avg_ms": 1.236, "count": 12, "max_ms": 1.489, "p95_ms": 1.489}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.566, "count": 12, "max_ms": 54.171, "p95_ms": 54.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.34, "count": 12, "max_ms": 64.309, "p95_ms": 64.309}, "yolo11_640_fire_11class": {"avg_ms": 49.466, "count": 12, "max_ms": 56.897, "p95_ms": 56.897}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.653, "count": 12, "max_ms": 64.322, "p95_ms": 64.322}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.822, "count": 12, "max_ms": 90.673, "p95_ms": 90.673}, "read_interval": {"avg_ms": 2736.195, "count": 13, "max_ms": 17878.482, "p95_ms": 17878.482}, "triton_wait": {"avg_ms": 52.713, "count": 12, "max_ms": 64.331, "p95_ms": 64.331}, "visualize": {"avg_ms": 3.417, "count": 12, "max_ms": 4.162, "p95_ms": 4.162}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.76, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10520.641, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:43:05+09:00", "last_error_summary": "[h264 @ 0x55b9e5647e80] error while decoding MB 9 4, bytestream -7 [in#0/rtsp @ 0x55b9e5604e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b9e5604e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 191, "h264_decode_error_count": 196, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:43:05+09:00", "last_error_summary": "[h264 @ 0x55b9e5647e80] error while decoding MB 9 4, bytestream -7 [in#0/rtsp @ 0x55b9e5604e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b9e5604e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:43:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 271.256, "last_started_at": "2026-07-01T04:38:34+09:00", "pid": 79068, "process": {"exists": false, "pid": 79068}, "restart_count": 190, "restart_count_per_hour": 12, "rtsp_timeout_count": 312, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 191}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.021, "sample_count": 2252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:43:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T04:43:05+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T04:43:05+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T04:43:05.231016+09:00", "mtime_epoch": 1782848585.231016, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107503}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:43:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1745.227, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.731, "pushed": 0.731, "read": 0.731}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.464, "count": 13, "max_ms": 24.702, "p95_ms": 24.702}, "gstreamer_push": {"avg_ms": 1.245, "count": 13, "max_ms": 1.489, "p95_ms": 1.489}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.055, "count": 13, "max_ms": 54.171, "p95_ms": 54.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.846, "count": 13, "max_ms": 64.309, "p95_ms": 64.309}, "yolo11_640_fire_11class": {"avg_ms": 49.192, "count": 13, "max_ms": 56.897, "p95_ms": 56.897}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.349, "count": 13, "max_ms": 64.322, "p95_ms": 64.322}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.031, "count": 13, "max_ms": 90.673, "p95_ms": 90.673}, "read_interval": {"avg_ms": 2736.195, "count": 13, "max_ms": 17878.482, "p95_ms": 17878.482}, "triton_wait": {"avg_ms": 52.328, "count": 13, "max_ms": 64.331, "p95_ms": 64.331}, "visualize": {"avg_ms": 3.609, "count": 13, "max_ms": 5.919, "p95_ms": 5.919}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.786, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10520.758, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 191, "h264_decode_error_count": 196, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:43:05+09:00", "last_error_summary": "[h264 @ 0x55b9e5647e80] error while decoding MB 9 4, bytestream -7 [in#0/rtsp @ 0x55b9e5604e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b9e5604e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:43:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 271.263, "last_started_at": "2026-07-01T04:38:34+09:00", "pid": 79068, "process": {"exists": false, "pid": 79068}, "restart_count": 190, "restart_count_per_hour": 12, "rtsp_timeout_count": 312, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 191}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.021, "sample_count": 2252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:43:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T04:43:05+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-01T04:43:05+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-01T04:43:05.231016+09:00", "mtime_epoch": 1782848585.231016, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107503}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:43:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1745.227, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.731, "pushed": 0.731, "read": 0.731}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.464, "count": 13, "max_ms": 24.702, "p95_ms": 24.702}, "gstreamer_push": {"avg_ms": 1.245, "count": 13, "max_ms": 1.489, "p95_ms": 1.489}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.055, "count": 13, "max_ms": 54.171, "p95_ms": 54.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.846, "count": 13, "max_ms": 64.309, "p95_ms": 64.309}, "yolo11_640_fire_11class": {"avg_ms": 49.192, "count": 13, "max_ms": 56.897, "p95_ms": 56.897}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.349, "count": 13, "max_ms": 64.322, "p95_ms": 64.322}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.031, "count": 13, "max_ms": 90.673, "p95_ms": 90.673}, "read_interval": {"avg_ms": 2736.195, "count": 13, "max_ms": 17878.482, "p95_ms": 17878.482}, "triton_wait": {"avg_ms": 52.328, "count": 13, "max_ms": 64.331, "p95_ms": 64.331}, "visualize": {"avg_ms": 3.609, "count": 13, "max_ms": 5.919, "p95_ms": 5.919}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.793, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10698.316, "limit_mb": 8796093022207.996}, "detail": {"pid": 79220, "restart_count": 191, "start_count": 192}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 191, "h264_decode_error_count": 196, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:43:05+09:00", "last_error_summary": "[h264 @ 0x55b9e5647e80] error while decoding MB 9 4, bytestream -7 [in#0/rtsp @ 0x55b9e5604e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b9e5604e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:43:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:43:08+09:00", "pid": 79220, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.121, "name": "ffmpeg", "pid": 79220, "rss_mb": 204.121, "state": "S (sleeping)", "threads": 106}, "restart_count": 191, "restart_count_per_hour": 13, "rtsp_timeout_count": 312, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 192}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.021, "sample_count": 2252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:43:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 5.019, "last_frame_pushed_at": "2026-07-01T04:43:05+09:00", "last_frame_read_age_sec": 5.104, "last_frame_read_at": "2026-07-01T04:43:05+09:00", "mjpeg": {"age_sec": 5.002, "exists": true, "mtime": "2026-07-01T04:43:05.327019+09:00", "mtime_epoch": 1782848585.3270192, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107802}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:43:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1745.227, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.57, "pushed": 0.57, "read": 0.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.464, "count": 13, "max_ms": 24.702, "p95_ms": 24.702}, "gstreamer_push": {"avg_ms": 1.245, "count": 13, "max_ms": 1.489, "p95_ms": 1.489}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.055, "count": 13, "max_ms": 54.171, "p95_ms": 54.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.846, "count": 13, "max_ms": 64.309, "p95_ms": 64.309}, "yolo11_640_fire_11class": {"avg_ms": 49.192, "count": 13, "max_ms": 56.897, "p95_ms": 56.897}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.349, "count": 13, "max_ms": 64.322, "p95_ms": 64.322}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.031, "count": 13, "max_ms": 90.673, "p95_ms": 90.673}, "read_interval": {"avg_ms": 2736.195, "count": 13, "max_ms": 17878.482, "p95_ms": 17878.482}, "triton_wait": {"avg_ms": 52.328, "count": 13, "max_ms": 64.331, "p95_ms": 64.331}, "visualize": {"avg_ms": 3.609, "count": 13, "max_ms": 5.919, "p95_ms": 5.919}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.797, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10978}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10551.918, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557c32796600] error while decoding MB 42 36, bytestream -11 [in#0/rtsp @ 0x557c319dde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c319dde00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 379, "h264_decode_error_count": 1243, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:46:14+09:00", "last_error_summary": "[h264 @ 0x557c32796600] error while decoding MB 42 36, bytestream -11 [in#0/rtsp @ 0x557c319dde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c319dde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:36:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 567.342, "last_started_at": "2026-07-01T04:36:47+09:00", "pid": 78961, "process": {"exists": false, "pid": 78961}, "restart_count": 412, "restart_count_per_hour": 6, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 413}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.526, "sample_count": 2586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1317433, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:46:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.085, "last_frame_pushed_at": "2026-07-01T04:46:14+09:00", "last_frame_read_age_sec": 0.073, "last_frame_read_at": "2026-07-01T04:46:14+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-01T04:46:14.329187+09:00", "mtime_epoch": 1782848774.3291867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116586}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:46:14+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4993.395, "name": "celery", "pid": 267, "rss_mb": 4993.395, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.202, "pushed": 3.202, "read": 3.202}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.408, "count": 96, "max_ms": 23.42, "p95_ms": 21.956}, "gstreamer_push": {"avg_ms": 1.197, "count": 96, "max_ms": 1.328, "p95_ms": 1.298}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.821, "count": 96, "max_ms": 44.019, "p95_ms": 41.282}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.075, "count": 96, "max_ms": 72.241, "p95_ms": 68.926}, "yolo11_640_fire_11class": {"avg_ms": 59.248, "count": 96, "max_ms": 69.291, "p95_ms": 66.058}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.669, "count": 96, "max_ms": 78.823, "p95_ms": 76.31}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.137, "count": 96, "max_ms": 106.905, "p95_ms": 105.16}, "read_interval": {"avg_ms": 312.562, "count": 96, "max_ms": 10123.484, "p95_ms": 121.343}, "triton_wait": {"avg_ms": 67.691, "count": 96, "max_ms": 78.835, "p95_ms": 76.32}, "visualize": {"avg_ms": 3.399, "count": 96, "max_ms": 3.614, "p95_ms": 3.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10978}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10551.844, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:46:14+09:00", "last_error_summary": "[h264 @ 0x557c32796600] error while decoding MB 42 36, bytestream -11 [in#0/rtsp @ 0x557c319dde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c319dde00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 379, "h264_decode_error_count": 1243, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:46:14+09:00", "last_error_summary": "[h264 @ 0x557c32796600] error while decoding MB 42 36, bytestream -11 [in#0/rtsp @ 0x557c319dde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c319dde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:46:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 567.368, "last_started_at": "2026-07-01T04:36:47+09:00", "pid": 78961, "process": {"exists": false, "pid": 78961}, "restart_count": 412, "restart_count_per_hour": 6, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 413}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.526, "sample_count": 2587, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1317434, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:46:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T04:46:14+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T04:46:14+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-01T04:46:14.329187+09:00", "mtime_epoch": 1782848774.3291867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116586}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:46:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4993.395, "name": "celery", "pid": 267, "rss_mb": 4993.395, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.201, "pushed": 3.201, "read": 3.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.476, "count": 96, "max_ms": 26.036, "p95_ms": 22.31}, "gstreamer_push": {"avg_ms": 1.199, "count": 96, "max_ms": 1.376, "p95_ms": 1.299}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.8, "count": 96, "max_ms": 44.019, "p95_ms": 41.282}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.959, "count": 96, "max_ms": 72.241, "p95_ms": 68.926}, "yolo11_640_fire_11class": {"avg_ms": 59.129, "count": 96, "max_ms": 69.291, "p95_ms": 66.058}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.463, "count": 96, "max_ms": 78.823, "p95_ms": 76.31}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.037, "count": 96, "max_ms": 106.905, "p95_ms": 105.16}, "read_interval": {"avg_ms": 312.562, "count": 96, "max_ms": 10123.484, "p95_ms": 121.343}, "triton_wait": {"avg_ms": 67.484, "count": 96, "max_ms": 78.835, "p95_ms": 76.32}, "visualize": {"avg_ms": 3.435, "count": 96, "max_ms": 6.853, "p95_ms": 3.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10978}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10551.613, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 379, "h264_decode_error_count": 1243, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:46:14+09:00", "last_error_summary": "[h264 @ 0x557c32796600] error while decoding MB 42 36, bytestream -11 [in#0/rtsp @ 0x557c319dde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c319dde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:46:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 567.377, "last_started_at": "2026-07-01T04:36:47+09:00", "pid": 78961, "process": {"exists": false, "pid": 78961}, "restart_count": 412, "restart_count_per_hour": 6, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 413}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.526, "sample_count": 2586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1317434, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:46:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T04:46:14+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-01T04:46:14+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-01T04:46:14.329187+09:00", "mtime_epoch": 1782848774.3291867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116586}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:46:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4993.395, "name": "celery", "pid": 267, "rss_mb": 4993.395, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.2, "pushed": 3.2, "read": 3.2}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.476, "count": 96, "max_ms": 26.036, "p95_ms": 22.31}, "gstreamer_push": {"avg_ms": 1.199, "count": 96, "max_ms": 1.376, "p95_ms": 1.299}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.8, "count": 96, "max_ms": 44.019, "p95_ms": 41.282}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.959, "count": 96, "max_ms": 72.241, "p95_ms": 68.926}, "yolo11_640_fire_11class": {"avg_ms": 59.129, "count": 96, "max_ms": 69.291, "p95_ms": 66.058}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.463, "count": 96, "max_ms": 78.823, "p95_ms": 76.31}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.037, "count": 96, "max_ms": 106.905, "p95_ms": 105.16}, "read_interval": {"avg_ms": 312.562, "count": 96, "max_ms": 10123.484, "p95_ms": 121.343}, "triton_wait": {"avg_ms": 67.484, "count": 96, "max_ms": 78.835, "p95_ms": 76.32}, "visualize": {"avg_ms": 3.435, "count": 96, "max_ms": 6.853, "p95_ms": 3.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 10978}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10659.719, "limit_mb": 8796093022207.996}, "detail": {"pid": 79372, "restart_count": 413, "start_count": 414}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 379, "h264_decode_error_count": 1243, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:46:14+09:00", "last_error_summary": "[h264 @ 0x557c32796600] error while decoding MB 42 36, bytestream -11 [in#0/rtsp @ 0x557c319dde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c319dde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:46:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:46:17+09:00", "pid": 79372, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.445, "name": "ffmpeg", "pid": 79372, "rss_mb": 138.445, "state": "S (sleeping)", "threads": 106}, "restart_count": 413, "restart_count_per_hour": 7, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 414}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 105.495, "sample_count": 2545, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1317434, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:46:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.869, "last_frame_pushed_at": "2026-07-01T04:46:14+09:00", "last_frame_read_age_sec": 4.962, "last_frame_read_at": "2026-07-01T04:46:14+09:00", "mjpeg": {"age_sec": 4.855, "exists": true, "mtime": "2026-07-01T04:46:14.433190+09:00", "mtime_epoch": 1782848774.4331896, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115828}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:46:19+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4993.395, "name": "celery", "pid": 267, "rss_mb": 4993.395, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 53, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.767, "pushed": 1.767, "read": 1.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.632, "count": 53, "max_ms": 26.036, "p95_ms": 23.005}, "gstreamer_push": {"avg_ms": 1.211, "count": 53, "max_ms": 1.376, "p95_ms": 1.316}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.038, "count": 53, "max_ms": 44.019, "p95_ms": 42.285}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.358, "count": 53, "max_ms": 72.241, "p95_ms": 68.969}, "yolo11_640_fire_11class": {"avg_ms": 58.22, "count": 53, "max_ms": 69.119, "p95_ms": 66.21}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.868, "count": 53, "max_ms": 78.823, "p95_ms": 76.537}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.626, "count": 53, "max_ms": 106.905, "p95_ms": 105.704}, "read_interval": {"avg_ms": 474.346, "count": 53, "max_ms": 10123.484, "p95_ms": 123.034}, "triton_wait": {"avg_ms": 65.895, "count": 53, "max_ms": 78.835, "p95_ms": 76.551}, "visualize": {"avg_ms": 3.445, "count": 53, "max_ms": 6.853, "p95_ms": 3.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12095}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10490.598, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d0e3761ac0] error while decoding MB 104 25, bytestream -5 [in#0/rtsp @ 0x55d0e29a7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d0e29a7e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 192, "h264_decode_error_count": 197, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:47:05+09:00", "last_error_summary": "[h264 @ 0x55d0e3761ac0] error while decoding MB 104 25, bytestream -5 [in#0/rtsp @ 0x55d0e29a7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d0e29a7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:43:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 237.658, "last_started_at": "2026-07-01T04:43:08+09:00", "pid": 79220, "process": {"exists": false, "pid": 79220}, "restart_count": 191, "restart_count_per_hour": 12, "rtsp_timeout_count": 314, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 192}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.454, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451467, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:47:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-01T04:47:05+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-01T04:47:05+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-01T04:47:05.930725+09:00", "mtime_epoch": 1782848825.9307249, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98771}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:47:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1746.0, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.728, "pushed": 0.728, "read": 0.784}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.651, "count": 13, "max_ms": 20.604, "p95_ms": 20.604}, "gstreamer_push": {"avg_ms": 1.188, "count": 13, "max_ms": 1.304, "p95_ms": 1.304}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.23, "count": 13, "max_ms": 57.637, "p95_ms": 57.637}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.009, "count": 13, "max_ms": 69.506, "p95_ms": 69.506}, "yolo11_640_fire_11class": {"avg_ms": 58.1, "count": 13, "max_ms": 69.519, "p95_ms": 69.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.615, "count": 13, "max_ms": 74.023, "p95_ms": 74.023}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.774, "count": 13, "max_ms": 100.754, "p95_ms": 100.754}, "read_interval": {"avg_ms": 2423.391, "count": 14, "max_ms": 16560.158, "p95_ms": 16560.158}, "triton_wait": {"avg_ms": 63.79, "count": 13, "max_ms": 74.036, "p95_ms": 74.036}, "visualize": {"avg_ms": 3.335, "count": 13, "max_ms": 3.475, "p95_ms": 3.475}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.86, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12095}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10490.238, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:47:05+09:00", "last_error_summary": "[h264 @ 0x55d0e3761ac0] error while decoding MB 104 25, bytestream -5 [in#0/rtsp @ 0x55d0e29a7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d0e29a7e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 192, "h264_decode_error_count": 197, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:47:05+09:00", "last_error_summary": "[h264 @ 0x55d0e3761ac0] error while decoding MB 104 25, bytestream -5 [in#0/rtsp @ 0x55d0e29a7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d0e29a7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:47:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 237.702, "last_started_at": "2026-07-01T04:43:08+09:00", "pid": 79220, "process": {"exists": false, "pid": 79220}, "restart_count": 191, "restart_count_per_hour": 12, "rtsp_timeout_count": 314, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 192}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.454, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451468, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:47:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T04:47:06+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-01T04:47:05+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-01T04:47:05.930725+09:00", "mtime_epoch": 1782848825.9307249, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98771}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:47:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1746.0, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.782, "pushed": 0.782, "read": 0.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.757, "count": 14, "max_ms": 20.604, "p95_ms": 20.604}, "gstreamer_push": {"avg_ms": 1.192, "count": 14, "max_ms": 1.304, "p95_ms": 1.304}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.977, "count": 14, "max_ms": 57.637, "p95_ms": 57.637}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.562, "count": 14, "max_ms": 69.506, "p95_ms": 69.506}, "yolo11_640_fire_11class": {"avg_ms": 58.636, "count": 14, "max_ms": 69.519, "p95_ms": 69.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.101, "count": 14, "max_ms": 74.023, "p95_ms": 74.023}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.418, "count": 14, "max_ms": 100.754, "p95_ms": 100.754}, "read_interval": {"avg_ms": 2423.391, "count": 14, "max_ms": 16560.158, "p95_ms": 16560.158}, "triton_wait": {"avg_ms": 64.193, "count": 14, "max_ms": 74.036, "p95_ms": 74.036}, "visualize": {"avg_ms": 3.354, "count": 14, "max_ms": 3.612, "p95_ms": 3.612}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.904, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12095}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10490.254, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 192, "h264_decode_error_count": 197, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:47:05+09:00", "last_error_summary": "[h264 @ 0x55d0e3761ac0] error while decoding MB 104 25, bytestream -5 [in#0/rtsp @ 0x55d0e29a7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d0e29a7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:47:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 237.709, "last_started_at": "2026-07-01T04:43:08+09:00", "pid": 79220, "process": {"exists": false, "pid": 79220}, "restart_count": 191, "restart_count_per_hour": 12, "rtsp_timeout_count": 314, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 192}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.454, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451468, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:47:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T04:47:06+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T04:47:05+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-01T04:47:05.930725+09:00", "mtime_epoch": 1782848825.9307249, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98771}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:47:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1746.0, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.782, "pushed": 0.782, "read": 0.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.757, "count": 14, "max_ms": 20.604, "p95_ms": 20.604}, "gstreamer_push": {"avg_ms": 1.192, "count": 14, "max_ms": 1.304, "p95_ms": 1.304}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.977, "count": 14, "max_ms": 57.637, "p95_ms": 57.637}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.562, "count": 14, "max_ms": 69.506, "p95_ms": 69.506}, "yolo11_640_fire_11class": {"avg_ms": 58.636, "count": 14, "max_ms": 69.519, "p95_ms": 69.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.101, "count": 14, "max_ms": 74.023, "p95_ms": 74.023}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.418, "count": 14, "max_ms": 100.754, "p95_ms": 100.754}, "read_interval": {"avg_ms": 2423.391, "count": 14, "max_ms": 16560.158, "p95_ms": 16560.158}, "triton_wait": {"avg_ms": 64.193, "count": 14, "max_ms": 74.036, "p95_ms": 74.036}, "visualize": {"avg_ms": 3.354, "count": 14, "max_ms": 3.612, "p95_ms": 3.612}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.911, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12095}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10514.816, "limit_mb": 8796093022207.996}, "detail": {"pid": 79479, "restart_count": 192, "start_count": 193}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 192, "h264_decode_error_count": 197, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:47:05+09:00", "last_error_summary": "[h264 @ 0x55d0e3761ac0] error while decoding MB 104 25, bytestream -5 [in#0/rtsp @ 0x55d0e29a7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d0e29a7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:47:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:47:08+09:00", "pid": 79479, "process": {"exists": true, "fd_count": 4, "hwm_mb": 59.02, "name": "ffmpeg", "pid": 79479, "rss_mb": 59.02, "state": "R (running)", "threads": 12}, "restart_count": 192, "restart_count_per_hour": 13, "rtsp_timeout_count": 314, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 193}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.456, "sample_count": 2059, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451468, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:47:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.954, "last_frame_pushed_at": "2026-07-01T04:47:06+09:00", "last_frame_read_age_sec": 5.056, "last_frame_read_at": "2026-07-01T04:47:05+09:00", "mjpeg": {"age_sec": 4.937, "exists": true, "mtime": "2026-07-01T04:47:06.042728+09:00", "mtime_epoch": 1782848826.0427282, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97747}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T04:47:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1746.0, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.613, "pushed": 0.613, "read": 0.613}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.757, "count": 14, "max_ms": 20.604, "p95_ms": 20.604}, "gstreamer_push": {"avg_ms": 1.192, "count": 14, "max_ms": 1.304, "p95_ms": 1.304}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.977, "count": 14, "max_ms": 57.637, "p95_ms": 57.637}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.562, "count": 14, "max_ms": 69.506, "p95_ms": 69.506}, "yolo11_640_fire_11class": {"avg_ms": 58.636, "count": 14, "max_ms": 69.519, "p95_ms": 69.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.101, "count": 14, "max_ms": 74.023, "p95_ms": 74.023}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.418, "count": 14, "max_ms": 100.754, "p95_ms": 100.754}, "read_interval": {"avg_ms": 2423.391, "count": 14, "max_ms": 16560.158, "p95_ms": 16560.158}, "triton_wait": {"avg_ms": 64.193, "count": 14, "max_ms": 74.036, "p95_ms": 74.036}, "visualize": {"avg_ms": 3.354, "count": 14, "max_ms": 3.612, "p95_ms": 3.612}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.853, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11027}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10556.312, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T04:46:14+09:00", "last_error_summary": "[h264 @ 0x557c32796600] error while decoding MB 42 36, bytestream -11 [in#0/rtsp @ 0x557c319dde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c319dde00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 379, "h264_decode_error_count": 1243, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:46:14+09:00", "last_error_summary": "[h264 @ 0x557c32796600] error while decoding MB 42 36, bytestream -11 [in#0/rtsp @ 0x557c319dde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c319dde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:56:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 641.624, "last_started_at": "2026-07-01T04:46:17+09:00", "pid": 79372, "process": {"exists": false, "pid": 79372}, "restart_count": 413, "restart_count_per_hour": 5, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 414}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.842, "sample_count": 2531, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1323286, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:56:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T04:56:58+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-01T04:56:58+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-01T04:56:58.785709+09:00", "mtime_epoch": 1782849418.7857087, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116552}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:56:58+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4996.195, "name": "celery", "pid": 267, "rss_mb": 4996.195, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 252, "pushed_frames": 252, "read_frames": 251, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.412, "pushed": 8.412, "read": 8.378}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.802, "count": 252, "max_ms": 35.971, "p95_ms": 25.3}, "gstreamer_push": {"avg_ms": 1.195, "count": 252, "max_ms": 1.446, "p95_ms": 1.328}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.476, "count": 252, "max_ms": 43.576, "p95_ms": 37.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.313, "count": 252, "max_ms": 82.392, "p95_ms": 79.951}, "yolo11_640_fire_11class": {"avg_ms": 54.063, "count": 252, "max_ms": 68.829, "p95_ms": 65.942}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.651, "count": 252, "max_ms": 79.631, "p95_ms": 77.472}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.267, "count": 252, "max_ms": 145.594, "p95_ms": 134.725}, "read_interval": {"avg_ms": 119.251, "count": 251, "max_ms": 158.484, "p95_ms": 147.549}, "triton_wait": {"avg_ms": 63.781, "count": 252, "max_ms": 82.404, "p95_ms": 79.967}, "visualize": {"avg_ms": 3.83, "count": 252, "max_ms": 6.171, "p95_ms": 4.806}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11027}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10556.43, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 379, "h264_decode_error_count": 1243, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:46:14+09:00", "last_error_summary": "[h264 @ 0x557c32796600] error while decoding MB 42 36, bytestream -11 [in#0/rtsp @ 0x557c319dde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c319dde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:56:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 641.63, "last_started_at": "2026-07-01T04:46:17+09:00", "pid": 79372, "process": {"exists": false, "pid": 79372}, "restart_count": 413, "restart_count_per_hour": 5, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 414}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.842, "sample_count": 2531, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1323286, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:56:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T04:56:58+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-01T04:56:58+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T04:56:58.785709+09:00", "mtime_epoch": 1782849418.7857087, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116552}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:56:58+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4996.195, "name": "celery", "pid": 267, "rss_mb": 4996.195, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 252, "pushed_frames": 252, "read_frames": 251, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.41, "pushed": 8.41, "read": 8.377}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.802, "count": 252, "max_ms": 35.971, "p95_ms": 25.3}, "gstreamer_push": {"avg_ms": 1.195, "count": 252, "max_ms": 1.446, "p95_ms": 1.328}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.476, "count": 252, "max_ms": 43.576, "p95_ms": 37.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.313, "count": 252, "max_ms": 82.392, "p95_ms": 79.951}, "yolo11_640_fire_11class": {"avg_ms": 54.063, "count": 252, "max_ms": 68.829, "p95_ms": 65.942}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.651, "count": 252, "max_ms": 79.631, "p95_ms": 77.472}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.267, "count": 252, "max_ms": 145.594, "p95_ms": 134.725}, "read_interval": {"avg_ms": 119.251, "count": 251, "max_ms": 158.484, "p95_ms": 147.549}, "triton_wait": {"avg_ms": 63.781, "count": 252, "max_ms": 82.404, "p95_ms": 79.967}, "visualize": {"avg_ms": 3.83, "count": 252, "max_ms": 6.171, "p95_ms": 4.806}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11027}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10661.68, "limit_mb": 8796093022207.996}, "detail": {"pid": 79631, "restart_count": 414, "start_count": 415}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 379, "h264_decode_error_count": 1243, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T04:46:14+09:00", "last_error_summary": "[h264 @ 0x557c32796600] error while decoding MB 42 36, bytestream -11 [in#0/rtsp @ 0x557c319dde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c319dde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:57:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T04:57:01+09:00", "pid": 79631, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.375, "name": "ffmpeg", "pid": 79631, "rss_mb": 136.375, "state": "S (sleeping)", "threads": 106}, "restart_count": 414, "restart_count_per_hour": 6, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 415}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.918, "sample_count": 2483, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1323286, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T04:56:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.525, "last_frame_pushed_at": "2026-07-01T04:56:58+09:00", "last_frame_read_age_sec": 4.65, "last_frame_read_at": "2026-07-01T04:56:58+09:00", "mjpeg": {"age_sec": 4.512, "exists": true, "mtime": "2026-07-01T04:56:58.917712+09:00", "mtime_epoch": 1782849418.9177117, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115965}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T04:57:03+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4996.195, "name": "celery", "pid": 267, "rss_mb": 4996.195, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.141, "pushed": 7.141, "read": 7.107}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.127, "count": 214, "max_ms": 35.971, "p95_ms": 25.439}, "gstreamer_push": {"avg_ms": 1.193, "count": 214, "max_ms": 1.446, "p95_ms": 1.337}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.441, "count": 214, "max_ms": 43.576, "p95_ms": 37.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.997, "count": 214, "max_ms": 81.188, "p95_ms": 79.492}, "yolo11_640_fire_11class": {"avg_ms": 52.363, "count": 214, "max_ms": 68.829, "p95_ms": 65.876}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.351, "count": 214, "max_ms": 78.784, "p95_ms": 77.472}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.391, "count": 214, "max_ms": 145.594, "p95_ms": 136.269}, "read_interval": {"avg_ms": 119.343, "count": 213, "max_ms": 158.484, "p95_ms": 149.488}, "triton_wait": {"avg_ms": 61.511, "count": 214, "max_ms": 81.201, "p95_ms": 79.505}, "visualize": {"avg_ms": 3.906, "count": 214, "max_ms": 6.171, "p95_ms": 4.839}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11039}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10558.797, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d0db572480] error while decoding MB 62 14, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 380, "h264_decode_error_count": 1244, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:00:16+09:00", "last_error_summary": "[h264 @ 0x55d0db572480] error while decoding MB 62 14, bytestream -5", "last_exit_at": "2026-07-01T04:57:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.35, "last_started_at": "2026-07-01T04:57:01+09:00", "pid": 79631, "process": {"exists": false, "pid": 79631}, "restart_count": 414, "restart_count_per_hour": 6, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 415}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.212, "sample_count": 2266, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1324707, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:00:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T05:00:16+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T05:00:16+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T05:00:16.734064+09:00", "mtime_epoch": 1782849616.734064, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:00:16+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4996.969, "name": "celery", "pid": 267, "rss_mb": 4996.969, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 218, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.27, "pushed": 7.27, "read": 7.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.984, "count": 217, "max_ms": 32.055, "p95_ms": 25.86}, "gstreamer_push": {"avg_ms": 1.121, "count": 218, "max_ms": 1.358, "p95_ms": 1.201}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.243, "count": 217, "max_ms": 40.911, "p95_ms": 36.406}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.881, "count": 217, "max_ms": 86.866, "p95_ms": 79.494}, "yolo11_640_fire_11class": {"avg_ms": 50.92, "count": 217, "max_ms": 69.95, "p95_ms": 62.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.563, "count": 217, "max_ms": 78.532, "p95_ms": 73.746}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.288, "count": 218, "max_ms": 166.464, "p95_ms": 144.763}, "read_interval": {"avg_ms": 137.899, "count": 218, "max_ms": 179.659, "p95_ms": 157.637}, "triton_wait": {"avg_ms": 60.983, "count": 217, "max_ms": 86.879, "p95_ms": 79.506}, "visualize": {"avg_ms": 4.619, "count": 217, "max_ms": 7.374, "p95_ms": 5.228}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11039}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10558.062, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:00:16+09:00", "last_error_summary": "[h264 @ 0x55d0db572480] error while decoding MB 62 14, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 380, "h264_decode_error_count": 1244, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:00:16+09:00", "last_error_summary": "[h264 @ 0x55d0db572480] error while decoding MB 62 14, bytestream -5", "last_exit_at": "2026-07-01T05:00:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.438, "last_started_at": "2026-07-01T04:57:01+09:00", "pid": 79631, "process": {"exists": false, "pid": 79631}, "restart_count": 414, "restart_count_per_hour": 6, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 415}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.212, "sample_count": 2267, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1324708, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:00:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T05:00:16+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-01T05:00:16+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-01T05:00:16.734064+09:00", "mtime_epoch": 1782849616.734064, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:00:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4996.969, "name": "celery", "pid": 267, "rss_mb": 4996.969, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.279, "pushed": 7.279, "read": 7.246}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.976, "count": 218, "max_ms": 32.055, "p95_ms": 25.86}, "gstreamer_push": {"avg_ms": 1.121, "count": 218, "max_ms": 1.358, "p95_ms": 1.201}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.251, "count": 218, "max_ms": 40.911, "p95_ms": 36.406}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.916, "count": 218, "max_ms": 86.866, "p95_ms": 79.494}, "yolo11_640_fire_11class": {"avg_ms": 50.961, "count": 218, "max_ms": 69.95, "p95_ms": 62.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.598, "count": 218, "max_ms": 78.532, "p95_ms": 73.746}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.304, "count": 218, "max_ms": 166.464, "p95_ms": 144.763}, "read_interval": {"avg_ms": 137.89, "count": 217, "max_ms": 179.659, "p95_ms": 157.637}, "triton_wait": {"avg_ms": 61.018, "count": 218, "max_ms": 86.879, "p95_ms": 79.506}, "visualize": {"avg_ms": 4.62, "count": 218, "max_ms": 7.374, "p95_ms": 5.228}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11039}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10558.18, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 380, "h264_decode_error_count": 1244, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:00:16+09:00", "last_error_summary": "[h264 @ 0x55d0db572480] error while decoding MB 62 14, bytestream -5", "last_exit_at": "2026-07-01T05:00:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.443, "last_started_at": "2026-07-01T04:57:01+09:00", "pid": 79631, "process": {"exists": false, "pid": 79631}, "restart_count": 414, "restart_count_per_hour": 6, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 415}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.212, "sample_count": 2267, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1324708, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:00:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T05:00:16+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-01T05:00:16+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-01T05:00:16.734064+09:00", "mtime_epoch": 1782849616.734064, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:00:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4996.969, "name": "celery", "pid": 267, "rss_mb": 4996.969, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.278, "pushed": 7.278, "read": 7.244}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.976, "count": 218, "max_ms": 32.055, "p95_ms": 25.86}, "gstreamer_push": {"avg_ms": 1.121, "count": 218, "max_ms": 1.358, "p95_ms": 1.201}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.251, "count": 218, "max_ms": 40.911, "p95_ms": 36.406}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.916, "count": 218, "max_ms": 86.866, "p95_ms": 79.494}, "yolo11_640_fire_11class": {"avg_ms": 50.961, "count": 218, "max_ms": 69.95, "p95_ms": 62.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.598, "count": 218, "max_ms": 78.532, "p95_ms": 73.746}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.304, "count": 218, "max_ms": 166.464, "p95_ms": 144.763}, "read_interval": {"avg_ms": 137.89, "count": 217, "max_ms": 179.659, "p95_ms": 157.637}, "triton_wait": {"avg_ms": 61.018, "count": 218, "max_ms": 86.879, "p95_ms": 79.506}, "visualize": {"avg_ms": 4.62, "count": 218, "max_ms": 7.374, "p95_ms": 5.228}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11039}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10668.535, "limit_mb": 8796093022207.996}, "detail": {"pid": 79738, "restart_count": 415, "start_count": 416}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 380, "h264_decode_error_count": 1244, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:00:16+09:00", "last_error_summary": "[h264 @ 0x55d0db572480] error while decoding MB 62 14, bytestream -5", "last_exit_at": "2026-07-01T05:00:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:00:19+09:00", "pid": 79738, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.504, "name": "ffmpeg", "pid": 79738, "rss_mb": 141.504, "state": "S (sleeping)", "threads": 106}, "restart_count": 415, "restart_count_per_hour": 7, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 416}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.18, "sample_count": 2233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1324708, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:00:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.547, "last_frame_pushed_at": "2026-07-01T05:00:16+09:00", "last_frame_read_age_sec": 4.68, "last_frame_read_at": "2026-07-01T05:00:16+09:00", "mjpeg": {"age_sec": 4.53, "exists": true, "mtime": "2026-07-01T05:00:16.878067+09:00", "mtime_epoch": 1782849616.8780673, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114291}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:00:21+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4996.969, "name": "celery", "pid": 267, "rss_mb": 4996.969, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 185, "pushed_frames": 185, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.188, "pushed": 6.188, "read": 6.154}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.899, "count": 185, "max_ms": 27.056, "p95_ms": 25.538}, "gstreamer_push": {"avg_ms": 1.122, "count": 185, "max_ms": 1.358, "p95_ms": 1.193}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.385, "count": 185, "max_ms": 40.911, "p95_ms": 36.272}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.765, "count": 185, "max_ms": 86.866, "p95_ms": 79.494}, "yolo11_640_fire_11class": {"avg_ms": 50.993, "count": 185, "max_ms": 69.95, "p95_ms": 62.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.513, "count": 185, "max_ms": 78.532, "p95_ms": 73.756}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.128, "count": 185, "max_ms": 166.464, "p95_ms": 144.763}, "read_interval": {"avg_ms": 137.687, "count": 184, "max_ms": 179.659, "p95_ms": 156.969}, "triton_wait": {"avg_ms": 60.883, "count": 185, "max_ms": 86.879, "p95_ms": 79.506}, "visualize": {"avg_ms": 4.615, "count": 185, "max_ms": 5.922, "p95_ms": 5.217}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11048}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10560.238, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55aa0ff4b200] error while decoding MB 31 8, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 381, "h264_decode_error_count": 1245, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:03:03+09:00", "last_error_summary": "[h264 @ 0x55aa0ff4b200] error while decoding MB 31 8, bytestream -5", "last_exit_at": "2026-07-01T05:00:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.745, "last_started_at": "2026-07-01T05:00:19+09:00", "pid": 79738, "process": {"exists": false, "pid": 79738}, "restart_count": 415, "restart_count_per_hour": 7, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 416}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.2, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1325819, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:03:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T05:03:03+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-01T05:03:03+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T05:03:03.105596+09:00", "mtime_epoch": 1782849783.1055963, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 104883}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:03:03+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4997.484, "name": "celery", "pid": 267, "rss_mb": 4997.484, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.763, "pushed": 6.763, "read": 6.763}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.511, "count": 202, "max_ms": 37.267, "p95_ms": 26.202}, "gstreamer_push": {"avg_ms": 1.089, "count": 202, "max_ms": 1.675, "p95_ms": 1.193}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.023, "count": 202, "max_ms": 43.719, "p95_ms": 37.478}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.64, "count": 202, "max_ms": 89.222, "p95_ms": 82.685}, "yolo11_640_fire_11class": {"avg_ms": 51.004, "count": 202, "max_ms": 74.491, "p95_ms": 64.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.476, "count": 202, "max_ms": 85.318, "p95_ms": 77.129}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.308, "count": 202, "max_ms": 176.668, "p95_ms": 165.502}, "read_interval": {"avg_ms": 148.4, "count": 202, "max_ms": 187.752, "p95_ms": 178.964}, "triton_wait": {"avg_ms": 62.76, "count": 202, "max_ms": 89.235, "p95_ms": 82.697}, "visualize": {"avg_ms": 5.02, "count": 202, "max_ms": 7.623, "p95_ms": 5.974}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11048}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10559.414, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:03:03+09:00", "last_error_summary": "[h264 @ 0x55aa0ff4b200] error while decoding MB 31 8, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 381, "h264_decode_error_count": 1245, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:03:03+09:00", "last_error_summary": "[h264 @ 0x55aa0ff4b200] error while decoding MB 31 8, bytestream -5", "last_exit_at": "2026-07-01T05:03:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.833, "last_started_at": "2026-07-01T05:00:19+09:00", "pid": 79738, "process": {"exists": false, "pid": 79738}, "restart_count": 415, "restart_count_per_hour": 7, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 416}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.2, "sample_count": 2093, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1325820, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:03:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T05:03:03+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-01T05:03:03+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T05:03:03.105596+09:00", "mtime_epoch": 1782849783.1055963, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 104883}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:03:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4997.484, "name": "celery", "pid": 267, "rss_mb": 4997.484, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.777, "pushed": 6.777, "read": 6.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.508, "count": 202, "max_ms": 37.267, "p95_ms": 26.202}, "gstreamer_push": {"avg_ms": 1.089, "count": 203, "max_ms": 1.675, "p95_ms": 1.193}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.028, "count": 202, "max_ms": 43.719, "p95_ms": 37.478}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.653, "count": 202, "max_ms": 89.222, "p95_ms": 82.685}, "yolo11_640_fire_11class": {"avg_ms": 51.02, "count": 202, "max_ms": 74.491, "p95_ms": 64.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.492, "count": 202, "max_ms": 85.318, "p95_ms": 77.129}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.274, "count": 203, "max_ms": 176.668, "p95_ms": 165.502}, "read_interval": {"avg_ms": 148.4, "count": 202, "max_ms": 187.752, "p95_ms": 178.964}, "triton_wait": {"avg_ms": 62.773, "count": 202, "max_ms": 89.235, "p95_ms": 82.697}, "visualize": {"avg_ms": 5.009, "count": 202, "max_ms": 7.623, "p95_ms": 5.956}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11048}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10559.406, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 381, "h264_decode_error_count": 1245, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:03:03+09:00", "last_error_summary": "[h264 @ 0x55aa0ff4b200] error while decoding MB 31 8, bytestream -5", "last_exit_at": "2026-07-01T05:03:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.838, "last_started_at": "2026-07-01T05:00:19+09:00", "pid": 79738, "process": {"exists": false, "pid": 79738}, "restart_count": 415, "restart_count_per_hour": 7, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 416}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.2, "sample_count": 2093, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1325820, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:03:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T05:03:03+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T05:03:03+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-01T05:03:03.105596+09:00", "mtime_epoch": 1782849783.1055963, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 104883}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:03:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4997.484, "name": "celery", "pid": 267, "rss_mb": 4997.484, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.776, "pushed": 6.776, "read": 6.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.508, "count": 202, "max_ms": 37.267, "p95_ms": 26.202}, "gstreamer_push": {"avg_ms": 1.089, "count": 203, "max_ms": 1.675, "p95_ms": 1.193}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.028, "count": 202, "max_ms": 43.719, "p95_ms": 37.478}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.653, "count": 202, "max_ms": 89.222, "p95_ms": 82.685}, "yolo11_640_fire_11class": {"avg_ms": 51.02, "count": 202, "max_ms": 74.491, "p95_ms": 64.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.492, "count": 202, "max_ms": 85.318, "p95_ms": 77.129}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.274, "count": 203, "max_ms": 176.668, "p95_ms": 165.502}, "read_interval": {"avg_ms": 148.4, "count": 202, "max_ms": 187.752, "p95_ms": 178.964}, "triton_wait": {"avg_ms": 62.773, "count": 202, "max_ms": 89.235, "p95_ms": 82.697}, "visualize": {"avg_ms": 5.009, "count": 202, "max_ms": 7.623, "p95_ms": 5.956}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11048}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10667.707, "limit_mb": 8796093022207.996}, "detail": {"pid": 79845, "restart_count": 416, "start_count": 417}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 381, "h264_decode_error_count": 1245, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:03:03+09:00", "last_error_summary": "[h264 @ 0x55aa0ff4b200] error while decoding MB 31 8, bytestream -5", "last_exit_at": "2026-07-01T05:03:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:03:05+09:00", "pid": 79845, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.203, "name": "ffmpeg", "pid": 79845, "rss_mb": 139.203, "state": "S (sleeping)", "threads": 106}, "restart_count": 416, "restart_count_per_hour": 8, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 417}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.363, "sample_count": 2063, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1325820, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:03:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.133, "last_frame_pushed_at": "2026-07-01T05:03:03+09:00", "last_frame_read_age_sec": 4.265, "last_frame_read_at": "2026-07-01T05:03:03+09:00", "mjpeg": {"age_sec": 4.119, "exists": true, "mtime": "2026-07-01T05:03:03.249599+09:00", "mtime_epoch": 1782849783.2495992, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 106417}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:03:07+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4997.484, "name": "celery", "pid": 267, "rss_mb": 4997.484, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.909, "pushed": 5.909, "read": 5.875}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.234, "count": 177, "max_ms": 27.209, "p95_ms": 25.808}, "gstreamer_push": {"avg_ms": 1.092, "count": 177, "max_ms": 1.28, "p95_ms": 1.192}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.181, "count": 177, "max_ms": 43.719, "p95_ms": 38.897}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.612, "count": 177, "max_ms": 89.222, "p95_ms": 82.691}, "yolo11_640_fire_11class": {"avg_ms": 51.253, "count": 177, "max_ms": 74.491, "p95_ms": 64.569}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.44, "count": 177, "max_ms": 85.318, "p95_ms": 78.079}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.546, "count": 177, "max_ms": 176.264, "p95_ms": 164.396}, "read_interval": {"avg_ms": 146.689, "count": 176, "max_ms": 187.752, "p95_ms": 175.509}, "triton_wait": {"avg_ms": 62.694, "count": 177, "max_ms": 89.235, "p95_ms": 82.705}, "visualize": {"avg_ms": 4.949, "count": 177, "max_ms": 6.576, "p95_ms": 5.921}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12177}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10498.84, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e195e11800] error while decoding MB 72 35, bytestream -7 [in#0/rtsp @ 0x55e19512fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e19512fe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 193, "h264_decode_error_count": 198, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:04:17+09:00", "last_error_summary": "[h264 @ 0x55e195e11800] error while decoding MB 72 35, bytestream -7 [in#0/rtsp @ 0x55e19512fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e19512fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T04:47:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 1028.979, "last_started_at": "2026-07-01T04:47:08+09:00", "pid": 79479, "process": {"exists": false, "pid": 79479}, "restart_count": 192, "restart_count_per_hour": 13, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 193}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.919, "sample_count": 2656, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1461307, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:04:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.089, "last_frame_pushed_at": "2026-07-01T05:04:17+09:00", "last_frame_read_age_sec": 0.075, "last_frame_read_at": "2026-07-01T05:04:17+09:00", "mjpeg": {"age_sec": 0.067, "exists": true, "mtime": "2026-07-01T05:04:17.891155+09:00", "mtime_epoch": 1782849857.8911555, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 112760}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:04:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1750.641, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.737, "pushed": 0.737, "read": 0.793}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.818, "count": 13, "max_ms": 20.668, "p95_ms": 20.668}, "gstreamer_push": {"avg_ms": 1.234, "count": 13, "max_ms": 1.321, "p95_ms": 1.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.12, "count": 13, "max_ms": 58.354, "p95_ms": 58.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.711, "count": 13, "max_ms": 80.775, "p95_ms": 80.775}, "yolo11_640_fire_11class": {"avg_ms": 49.192, "count": 13, "max_ms": 61.366, "p95_ms": 61.366}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.062, "count": 13, "max_ms": 75.818, "p95_ms": 75.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.596, "count": 13, "max_ms": 107.805, "p95_ms": 107.805}, "read_interval": {"avg_ms": 2275.995, "count": 14, "max_ms": 16375.968, "p95_ms": 16375.968}, "triton_wait": {"avg_ms": 59.723, "count": 13, "max_ms": 80.787, "p95_ms": 80.787}, "visualize": {"avg_ms": 3.447, "count": 13, "max_ms": 3.582, "p95_ms": 3.582}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.646, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12177}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10498.66, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:04:17+09:00", "last_error_summary": "[h264 @ 0x55e195e11800] error while decoding MB 72 35, bytestream -7 [in#0/rtsp @ 0x55e19512fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e19512fe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 193, "h264_decode_error_count": 198, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:04:17+09:00", "last_error_summary": "[h264 @ 0x55e195e11800] error while decoding MB 72 35, bytestream -7 [in#0/rtsp @ 0x55e19512fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e19512fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:04:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 1029.003, "last_started_at": "2026-07-01T04:47:08+09:00", "pid": 79479, "process": {"exists": false, "pid": 79479}, "restart_count": 192, "restart_count_per_hour": 13, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 193}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.919, "sample_count": 2656, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1461308, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:04:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T05:04:17+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-01T05:04:17+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T05:04:17.891155+09:00", "mtime_epoch": 1782849857.8911555, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 112760}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:04:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1750.641, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.792, "pushed": 0.792, "read": 0.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.092, "count": 14, "max_ms": 22.644, "p95_ms": 22.644}, "gstreamer_push": {"avg_ms": 1.235, "count": 14, "max_ms": 1.321, "p95_ms": 1.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.754, "count": 14, "max_ms": 58.354, "p95_ms": 58.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.538, "count": 14, "max_ms": 80.775, "p95_ms": 80.775}, "yolo11_640_fire_11class": {"avg_ms": 49.204, "count": 14, "max_ms": 61.366, "p95_ms": 61.366}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.837, "count": 14, "max_ms": 75.818, "p95_ms": 75.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.899, "count": 14, "max_ms": 107.805, "p95_ms": 107.805}, "read_interval": {"avg_ms": 2275.995, "count": 14, "max_ms": 16375.968, "p95_ms": 16375.968}, "triton_wait": {"avg_ms": 59.551, "count": 14, "max_ms": 80.787, "p95_ms": 80.787}, "visualize": {"avg_ms": 3.638, "count": 14, "max_ms": 6.119, "p95_ms": 6.119}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.67, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12177}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10498.523, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 193, "h264_decode_error_count": 198, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:04:17+09:00", "last_error_summary": "[h264 @ 0x55e195e11800] error while decoding MB 72 35, bytestream -7 [in#0/rtsp @ 0x55e19512fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e19512fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:04:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 1029.011, "last_started_at": "2026-07-01T04:47:08+09:00", "pid": 79479, "process": {"exists": false, "pid": 79479}, "restart_count": 192, "restart_count_per_hour": 13, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 193}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.919, "sample_count": 2656, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1461308, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:04:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T05:04:17+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-01T05:04:17+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-01T05:04:17.891155+09:00", "mtime_epoch": 1782849857.8911555, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 112760}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:04:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1750.641, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.792, "pushed": 0.792, "read": 0.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.092, "count": 14, "max_ms": 22.644, "p95_ms": 22.644}, "gstreamer_push": {"avg_ms": 1.235, "count": 14, "max_ms": 1.321, "p95_ms": 1.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.754, "count": 14, "max_ms": 58.354, "p95_ms": 58.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.538, "count": 14, "max_ms": 80.775, "p95_ms": 80.775}, "yolo11_640_fire_11class": {"avg_ms": 49.204, "count": 14, "max_ms": 61.366, "p95_ms": 61.366}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.837, "count": 14, "max_ms": 75.818, "p95_ms": 75.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.899, "count": 14, "max_ms": 107.805, "p95_ms": 107.805}, "read_interval": {"avg_ms": 2275.995, "count": 14, "max_ms": 16375.968, "p95_ms": 16375.968}, "triton_wait": {"avg_ms": 59.551, "count": 14, "max_ms": 80.787, "p95_ms": 80.787}, "visualize": {"avg_ms": 3.638, "count": 14, "max_ms": 6.119, "p95_ms": 6.119}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.677, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12177}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10519.883, "limit_mb": 8796093022207.996}, "detail": {"pid": 79952, "restart_count": 193, "start_count": 194}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 193, "h264_decode_error_count": 198, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:04:17+09:00", "last_error_summary": "[h264 @ 0x55e195e11800] error while decoding MB 72 35, bytestream -7 [in#0/rtsp @ 0x55e19512fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e19512fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:04:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:04:20+09:00", "pid": 79952, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.094, "name": "ffmpeg", "pid": 79952, "rss_mb": 57.094, "state": "S (sleeping)", "threads": 1}, "restart_count": 193, "restart_count_per_hour": 14, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 194}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.976, "sample_count": 2610, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1461308, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:04:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.691, "last_frame_pushed_at": "2026-07-01T05:04:17+09:00", "last_frame_read_age_sec": 4.783, "last_frame_read_at": "2026-07-01T05:04:17+09:00", "mjpeg": {"age_sec": 4.676, "exists": true, "mtime": "2026-07-01T05:04:17.991158+09:00", "mtime_epoch": 1782849857.9911575, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 110403}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:04:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1750.641, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.626, "pushed": 0.626, "read": 0.626}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.092, "count": 14, "max_ms": 22.644, "p95_ms": 22.644}, "gstreamer_push": {"avg_ms": 1.235, "count": 14, "max_ms": 1.321, "p95_ms": 1.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.754, "count": 14, "max_ms": 58.354, "p95_ms": 58.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.538, "count": 14, "max_ms": 80.775, "p95_ms": 80.775}, "yolo11_640_fire_11class": {"avg_ms": 49.204, "count": 14, "max_ms": 61.366, "p95_ms": 61.366}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.837, "count": 14, "max_ms": 75.818, "p95_ms": 75.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.899, "count": 14, "max_ms": 107.805, "p95_ms": 107.805}, "read_interval": {"avg_ms": 2275.995, "count": 14, "max_ms": 16375.968, "p95_ms": 16375.968}, "triton_wait": {"avg_ms": 59.551, "count": 14, "max_ms": 80.787, "p95_ms": 80.787}, "visualize": {"avg_ms": 3.638, "count": 14, "max_ms": 6.119, "p95_ms": 6.119}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.355, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10563.598, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c32b7d07c0] error while decoding MB 37 30, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 382, "h264_decode_error_count": 1246, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:05:47+09:00", "last_error_summary": "[h264 @ 0x55c32b7d07c0] error while decoding MB 37 30, bytestream -5", "last_exit_at": "2026-07-01T05:03:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 161.944, "last_started_at": "2026-07-01T05:03:05+09:00", "pid": 79845, "process": {"exists": false, "pid": 79845}, "restart_count": 416, "restart_count_per_hour": 8, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 417}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.551, "sample_count": 1988, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1326874, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:05:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T05:05:47+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T05:05:47+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T05:05:47.269000+09:00", "mtime_epoch": 1782849947.269, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121328}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:05:47+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4998.0, "name": "celery", "pid": 267, "rss_mb": 4998.0, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.352, "pushed": 6.352, "read": 6.352}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.831, "count": 190, "max_ms": 30.273, "p95_ms": 26.214}, "gstreamer_push": {"avg_ms": 1.044, "count": 190, "max_ms": 1.233, "p95_ms": 1.17}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.998, "count": 190, "max_ms": 43.281, "p95_ms": 35.618}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.011, "count": 190, "max_ms": 93.283, "p95_ms": 81.001}, "yolo11_640_fire_11class": {"avg_ms": 48.987, "count": 190, "max_ms": 72.611, "p95_ms": 61.656}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.793, "count": 190, "max_ms": 84.232, "p95_ms": 77.933}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.439, "count": 190, "max_ms": 183.601, "p95_ms": 174.47}, "read_interval": {"avg_ms": 157.769, "count": 190, "max_ms": 194.913, "p95_ms": 185.115}, "triton_wait": {"avg_ms": 62.197, "count": 190, "max_ms": 93.295, "p95_ms": 81.013}, "visualize": {"avg_ms": 5.477, "count": 190, "max_ms": 7.767, "p95_ms": 6.686}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10563.207, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:05:47+09:00", "last_error_summary": "[h264 @ 0x55c32b7d07c0] error while decoding MB 37 30, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 382, "h264_decode_error_count": 1246, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:05:47+09:00", "last_error_summary": "[h264 @ 0x55c32b7d07c0] error while decoding MB 37 30, bytestream -5", "last_exit_at": "2026-07-01T05:05:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.06, "last_started_at": "2026-07-01T05:03:05+09:00", "pid": 79845, "process": {"exists": false, "pid": 79845}, "restart_count": 416, "restart_count_per_hour": 8, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 417}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.551, "sample_count": 1988, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1326875, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:05:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T05:05:47+09:00", "last_frame_read_age_sec": 0.166, "last_frame_read_at": "2026-07-01T05:05:47+09:00", "mjpeg": {"age_sec": 0.159, "exists": true, "mtime": "2026-07-01T05:05:47.269000+09:00", "mtime_epoch": 1782849947.269, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121328}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:05:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4998.0, "name": "celery", "pid": 267, "rss_mb": 4998.0, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.358, "pushed": 6.358, "read": 6.325}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.841, "count": 190, "max_ms": 30.273, "p95_ms": 26.214}, "gstreamer_push": {"avg_ms": 1.043, "count": 190, "max_ms": 1.233, "p95_ms": 1.17}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.007, "count": 190, "max_ms": 43.281, "p95_ms": 35.618}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.09, "count": 190, "max_ms": 93.283, "p95_ms": 81.001}, "yolo11_640_fire_11class": {"avg_ms": 49.062, "count": 190, "max_ms": 72.611, "p95_ms": 61.656}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.89, "count": 190, "max_ms": 84.232, "p95_ms": 77.933}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.668, "count": 190, "max_ms": 183.601, "p95_ms": 174.47}, "read_interval": {"avg_ms": 157.921, "count": 189, "max_ms": 194.913, "p95_ms": 185.115}, "triton_wait": {"avg_ms": 62.277, "count": 190, "max_ms": 93.295, "p95_ms": 81.013}, "visualize": {"avg_ms": 5.484, "count": 190, "max_ms": 7.767, "p95_ms": 6.686}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10563.324, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 382, "h264_decode_error_count": 1246, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:05:47+09:00", "last_error_summary": "[h264 @ 0x55c32b7d07c0] error while decoding MB 37 30, bytestream -5", "last_exit_at": "2026-07-01T05:05:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.065, "last_started_at": "2026-07-01T05:03:05+09:00", "pid": 79845, "process": {"exists": false, "pid": 79845}, "restart_count": 416, "restart_count_per_hour": 8, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 417}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.551, "sample_count": 1988, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1326875, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:05:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T05:05:47+09:00", "last_frame_read_age_sec": 0.171, "last_frame_read_at": "2026-07-01T05:05:47+09:00", "mjpeg": {"age_sec": 0.163, "exists": true, "mtime": "2026-07-01T05:05:47.269000+09:00", "mtime_epoch": 1782849947.269, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121328}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:05:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4998.0, "name": "celery", "pid": 267, "rss_mb": 4998.0, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.357, "pushed": 6.357, "read": 6.324}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.841, "count": 190, "max_ms": 30.273, "p95_ms": 26.214}, "gstreamer_push": {"avg_ms": 1.043, "count": 190, "max_ms": 1.233, "p95_ms": 1.17}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.007, "count": 190, "max_ms": 43.281, "p95_ms": 35.618}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.09, "count": 190, "max_ms": 93.283, "p95_ms": 81.001}, "yolo11_640_fire_11class": {"avg_ms": 49.062, "count": 190, "max_ms": 72.611, "p95_ms": 61.656}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.89, "count": 190, "max_ms": 84.232, "p95_ms": 77.933}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.668, "count": 190, "max_ms": 183.601, "p95_ms": 174.47}, "read_interval": {"avg_ms": 157.921, "count": 189, "max_ms": 194.913, "p95_ms": 185.115}, "triton_wait": {"avg_ms": 62.277, "count": 190, "max_ms": 93.295, "p95_ms": 81.013}, "visualize": {"avg_ms": 5.484, "count": 190, "max_ms": 7.767, "p95_ms": 6.686}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.886, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10671.457, "limit_mb": 8796093022207.996}, "detail": {"pid": 80104, "restart_count": 417, "start_count": 418}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 382, "h264_decode_error_count": 1246, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:05:47+09:00", "last_error_summary": "[h264 @ 0x55c32b7d07c0] error while decoding MB 37 30, bytestream -5", "last_exit_at": "2026-07-01T05:05:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:05:50+09:00", "pid": 80104, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.43, "name": "ffmpeg", "pid": 80104, "rss_mb": 138.43, "state": "S (sleeping)", "threads": 106}, "restart_count": 417, "restart_count_per_hour": 9, "rtsp_timeout_count": 374, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 418}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.551, "sample_count": 1956, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1326875, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:05:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.974, "last_frame_pushed_at": "2026-07-01T05:05:47+09:00", "last_frame_read_age_sec": 5.137, "last_frame_read_at": "2026-07-01T05:05:47+09:00", "mjpeg": {"age_sec": 4.961, "exists": true, "mtime": "2026-07-01T05:05:47.437004+09:00", "mtime_epoch": 1782849947.4370036, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120907}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:05:52+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4998.0, "name": "celery", "pid": 267, "rss_mb": 4998.0, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 159, "pushed_frames": 159, "read_frames": 158, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.302, "pushed": 5.302, "read": 5.269}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 23.938, "count": 158, "max_ms": 30.273, "p95_ms": 26.397}, "gstreamer_push": {"avg_ms": 1.044, "count": 159, "max_ms": 1.219, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.318, "count": 158, "max_ms": 43.281, "p95_ms": 37.946}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.03, "count": 158, "max_ms": 93.283, "p95_ms": 80.529}, "yolo11_640_fire_11class": {"avg_ms": 48.891, "count": 158, "max_ms": 67.134, "p95_ms": 61.634}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.082, "count": 158, "max_ms": 84.232, "p95_ms": 77.933}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.009, "count": 159, "max_ms": 183.601, "p95_ms": 174.506}, "read_interval": {"avg_ms": 158.284, "count": 158, "max_ms": 194.913, "p95_ms": 185.433}, "triton_wait": {"avg_ms": 62.252, "count": 158, "max_ms": 93.295, "p95_ms": 80.54}, "visualize": {"avg_ms": 5.489, "count": 158, "max_ms": 7.767, "p95_ms": 6.745}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11065}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10565.0, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561c4a536240] left block unavailable for requested intra mode [h264 @ 0x561c4a536240] error while decoding MB 0 43, bytestream 46 [h264 @ 0x561c4a607e80] left block unavailable for requested intra mode [h264 @ 0x561c4a607e80] error while decoding MB 0 13, bytestream 6265 [h264 @ 0x561c4ae5ce80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c4ae5ce80] error while decoding MB 0 17, bytestream 3438 [in#0/rtsp @ 0x561c4a4b2e00] Error during demuxing: Connection timed o", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 383, "h264_decode_error_count": 1249, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:08:47+09:00", "last_error_summary": "[h264 @ 0x561c4a536240] left block unavailable for requested intra mode [h264 @ 0x561c4a536240] error while decoding MB 0 43, bytestream 46 [h264 @ 0x561c4a607e80] left block unavailable for requested intra mode [h264 @ 0x561c4a607e80] error while decoding MB 0 13, bytestream 6265 [h264 @ 0x561c4ae5ce80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c4ae5ce80] error while decoding MB 0 17, bytestream 3438 [in#0/rtsp @ 0x561c4a4b2e00] Error during demuxing: Connection timed o", "last_exit_at": "2026-07-01T05:05:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 176.642, "last_started_at": "2026-07-01T05:05:50+09:00", "pid": 80104, "process": {"exists": false, "pid": 80104}, "restart_count": 417, "restart_count_per_hour": 9, "rtsp_timeout_count": 376, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 418}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.605, "sample_count": 1730, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1327809, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:08:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T05:08:46+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T05:08:46+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T05:08:46.992660+09:00", "mtime_epoch": 1782850126.9926603, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114184}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:08:47+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4998.516, "name": "celery", "pid": 267, "rss_mb": 4998.516, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 73, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.435, "pushed": 2.435, "read": 2.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.655, "count": 72, "max_ms": 27.36, "p95_ms": 25.672}, "gstreamer_push": {"avg_ms": 1.068, "count": 73, "max_ms": 1.836, "p95_ms": 1.169}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.377, "count": 72, "max_ms": 36.236, "p95_ms": 34.945}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.388, "count": 72, "max_ms": 87.15, "p95_ms": 82.084}, "yolo11_640_fire_11class": {"avg_ms": 51.911, "count": 72, "max_ms": 70.819, "p95_ms": 62.215}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.86, "count": 72, "max_ms": 82.24, "p95_ms": 78.658}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.095, "count": 73, "max_ms": 177.178, "p95_ms": 171.753}, "read_interval": {"avg_ms": 412.149, "count": 73, "max_ms": 19119.744, "p95_ms": 184.031}, "triton_wait": {"avg_ms": 61.771, "count": 72, "max_ms": 87.162, "p95_ms": 82.097}, "visualize": {"avg_ms": 5.235, "count": 72, "max_ms": 6.885, "p95_ms": 6.526}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11065}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10564.586, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:08:47+09:00", "last_error_summary": "[h264 @ 0x561c4a536240] left block unavailable for requested intra mode [h264 @ 0x561c4a536240] error while decoding MB 0 43, bytestream 46 [h264 @ 0x561c4a607e80] left block unavailable for requested intra mode [h264 @ 0x561c4a607e80] error while decoding MB 0 13, bytestream 6265 [h264 @ 0x561c4ae5ce80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c4ae5ce80] error while decoding MB 0 17, bytestream 3438 [in#0/rtsp @ 0x561c4a4b2e00] Error during demuxing: Connection timed o", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 383, "h264_decode_error_count": 1249, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:08:47+09:00", "last_error_summary": "[h264 @ 0x561c4a536240] left block unavailable for requested intra mode [h264 @ 0x561c4a536240] error while decoding MB 0 43, bytestream 46 [h264 @ 0x561c4a607e80] left block unavailable for requested intra mode [h264 @ 0x561c4a607e80] error while decoding MB 0 13, bytestream 6265 [h264 @ 0x561c4ae5ce80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c4ae5ce80] error while decoding MB 0 17, bytestream 3438 [in#0/rtsp @ 0x561c4a4b2e00] Error during demuxing: Connection timed o", "last_exit_at": "2026-07-01T05:08:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 176.776, "last_started_at": "2026-07-01T05:05:50+09:00", "pid": 80104, "process": {"exists": false, "pid": 80104}, "restart_count": 417, "restart_count_per_hour": 9, "rtsp_timeout_count": 376, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 418}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.652, "sample_count": 1730, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1327810, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:08:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T05:08:47+09:00", "last_frame_read_age_sec": 0.187, "last_frame_read_at": "2026-07-01T05:08:46+09:00", "mjpeg": {"age_sec": 0.181, "exists": true, "mtime": "2026-07-01T05:08:46.992660+09:00", "mtime_epoch": 1782850126.9926603, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114184}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:08:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4998.516, "name": "celery", "pid": 267, "rss_mb": 4998.516, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.437, "pushed": 2.437, "read": 2.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.686, "count": 72, "max_ms": 27.36, "p95_ms": 25.672}, "gstreamer_push": {"avg_ms": 1.066, "count": 73, "max_ms": 1.836, "p95_ms": 1.169}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.354, "count": 72, "max_ms": 36.236, "p95_ms": 34.945}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.601, "count": 72, "max_ms": 87.15, "p95_ms": 82.084}, "yolo11_640_fire_11class": {"avg_ms": 52.005, "count": 72, "max_ms": 70.819, "p95_ms": 62.215}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.959, "count": 72, "max_ms": 82.24, "p95_ms": 78.658}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.368, "count": 73, "max_ms": 182.274, "p95_ms": 172.88}, "read_interval": {"avg_ms": 415.472, "count": 72, "max_ms": 19119.744, "p95_ms": 184.031}, "triton_wait": {"avg_ms": 61.984, "count": 72, "max_ms": 87.162, "p95_ms": 82.097}, "visualize": {"avg_ms": 5.249, "count": 72, "max_ms": 6.885, "p95_ms": 6.591}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11065}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10564.602, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 383, "h264_decode_error_count": 1249, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:08:47+09:00", "last_error_summary": "[h264 @ 0x561c4a536240] left block unavailable for requested intra mode [h264 @ 0x561c4a536240] error while decoding MB 0 43, bytestream 46 [h264 @ 0x561c4a607e80] left block unavailable for requested intra mode [h264 @ 0x561c4a607e80] error while decoding MB 0 13, bytestream 6265 [h264 @ 0x561c4ae5ce80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c4ae5ce80] error while decoding MB 0 17, bytestream 3438 [in#0/rtsp @ 0x561c4a4b2e00] Error during demuxing: Connection timed o", "last_exit_at": "2026-07-01T05:08:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 176.78, "last_started_at": "2026-07-01T05:05:50+09:00", "pid": 80104, "process": {"exists": false, "pid": 80104}, "restart_count": 417, "restart_count_per_hour": 9, "rtsp_timeout_count": 376, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 418}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.652, "sample_count": 1730, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1327810, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:08:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T05:08:47+09:00", "last_frame_read_age_sec": 0.19, "last_frame_read_at": "2026-07-01T05:08:46+09:00", "mjpeg": {"age_sec": 0.184, "exists": true, "mtime": "2026-07-01T05:08:46.992660+09:00", "mtime_epoch": 1782850126.9926603, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114184}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:08:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4998.516, "name": "celery", "pid": 267, "rss_mb": 4998.516, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.437, "pushed": 2.437, "read": 2.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.686, "count": 72, "max_ms": 27.36, "p95_ms": 25.672}, "gstreamer_push": {"avg_ms": 1.066, "count": 73, "max_ms": 1.836, "p95_ms": 1.169}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.354, "count": 72, "max_ms": 36.236, "p95_ms": 34.945}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.601, "count": 72, "max_ms": 87.15, "p95_ms": 82.084}, "yolo11_640_fire_11class": {"avg_ms": 52.005, "count": 72, "max_ms": 70.819, "p95_ms": 62.215}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.959, "count": 72, "max_ms": 82.24, "p95_ms": 78.658}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.368, "count": 73, "max_ms": 182.274, "p95_ms": 172.88}, "read_interval": {"avg_ms": 415.472, "count": 72, "max_ms": 19119.744, "p95_ms": 184.031}, "triton_wait": {"avg_ms": 61.984, "count": 72, "max_ms": 87.162, "p95_ms": 82.097}, "visualize": {"avg_ms": 5.249, "count": 72, "max_ms": 6.885, "p95_ms": 6.591}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11065}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10672.293, "limit_mb": 8796093022207.996}, "detail": {"pid": 80211, "restart_count": 418, "start_count": 419}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 383, "h264_decode_error_count": 1249, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:08:47+09:00", "last_error_summary": "[h264 @ 0x561c4a536240] left block unavailable for requested intra mode [h264 @ 0x561c4a536240] error while decoding MB 0 43, bytestream 46 [h264 @ 0x561c4a607e80] left block unavailable for requested intra mode [h264 @ 0x561c4a607e80] error while decoding MB 0 13, bytestream 6265 [h264 @ 0x561c4ae5ce80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561c4ae5ce80] error while decoding MB 0 17, bytestream 3438 [in#0/rtsp @ 0x561c4a4b2e00] Error during demuxing: Connection timed o", "last_exit_at": "2026-07-01T05:08:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:08:50+09:00", "pid": 80211, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.574, "name": "ffmpeg", "pid": 80211, "rss_mb": 138.574, "state": "S (sleeping)", "threads": 106}, "restart_count": 418, "restart_count_per_hour": 10, "rtsp_timeout_count": 376, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 419}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.718, "sample_count": 1695, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1327810, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:08:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.916, "last_frame_pushed_at": "2026-07-01T05:08:47+09:00", "last_frame_read_age_sec": 5.1, "last_frame_read_at": "2026-07-01T05:08:46+09:00", "mjpeg": {"age_sec": 4.901, "exists": true, "mtime": "2026-07-01T05:08:47.184664+09:00", "mtime_epoch": 1782850127.1846642, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115089}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:08:52+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4998.516, "name": "celery", "pid": 267, "rss_mb": 4998.516, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 38, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.303, "pushed": 1.303, "read": 1.269}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.75, "count": 39, "max_ms": 27.36, "p95_ms": 26.258}, "gstreamer_push": {"avg_ms": 1.021, "count": 39, "max_ms": 1.165, "p95_ms": 1.164}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.208, "count": 39, "max_ms": 34.945, "p95_ms": 34.505}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.792, "count": 39, "max_ms": 84.294, "p95_ms": 82.084}, "yolo11_640_fire_11class": {"avg_ms": 51.221, "count": 39, "max_ms": 63.169, "p95_ms": 61.202}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.177, "count": 39, "max_ms": 78.658, "p95_ms": 77.512}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.212, "count": 39, "max_ms": 182.274, "p95_ms": 177.178}, "read_interval": {"avg_ms": 657.066, "count": 38, "max_ms": 19119.744, "p95_ms": 187.533}, "triton_wait": {"avg_ms": 62.452, "count": 39, "max_ms": 84.306, "p95_ms": 82.097}, "visualize": {"avg_ms": 5.528, "count": 39, "max_ms": 6.885, "p95_ms": 6.754}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10569.461, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 10, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d782d55c00] non-existing PPS 3 referenced [h264 @ 0x55d782d55c00] decode_slice_header error [h264 @ 0x55d783160480] cabac decode of qscale diff failed at 41 41 [h264 @ 0x55d783160480] error while decoding MB 41 41, bytestream 30590 [h264 @ 0x55d7824dc340] cabac decode of qscale diff failed at 29 21 [h264 @ 0x55d7824dc340] error while decoding MB 29 21, bytestream 5000 [h264 @ 0x55d782421e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d782421e00] error while dec", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 384, "h264_decode_error_count": 1259, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:13:15+09:00", "last_error_summary": "[h264 @ 0x55d782d55c00] non-existing PPS 3 referenced [h264 @ 0x55d782d55c00] decode_slice_header error [h264 @ 0x55d783160480] cabac decode of qscale diff failed at 41 41 [h264 @ 0x55d783160480] error while decoding MB 41 41, bytestream 30590 [h264 @ 0x55d7824dc340] cabac decode of qscale diff failed at 29 21 [h264 @ 0x55d7824dc340] error while decoding MB 29 21, bytestream 5000 [h264 @ 0x55d782421e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d782421e00] error while dec", "last_exit_at": "2026-07-01T05:08:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 265.044, "last_started_at": "2026-07-01T05:08:50+09:00", "pid": 80211, "process": {"exists": false, "pid": 80211}, "restart_count": 418, "restart_count_per_hour": 9, "rtsp_timeout_count": 378, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 419}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.718, "sample_count": 2181, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1329906, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:13:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.084, "last_frame_pushed_at": "2026-07-01T05:13:15+09:00", "last_frame_read_age_sec": 0.073, "last_frame_read_at": "2026-07-01T05:13:15+09:00", "mjpeg": {"age_sec": 0.067, "exists": true, "mtime": "2026-07-01T05:13:15.062032+09:00", "mtime_epoch": 1782850395.0620325, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 142255}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:13:15+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4999.547, "name": "celery", "pid": 267, "rss_mb": 4999.547, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.371, "pushed": 3.371, "read": 3.371}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.37, "count": 100, "max_ms": 25.301, "p95_ms": 22.521}, "gstreamer_push": {"avg_ms": 1.186, "count": 101, "max_ms": 1.639, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.365, "count": 100, "max_ms": 55.615, "p95_ms": 32.84}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.15, "count": 100, "max_ms": 79.881, "p95_ms": 60.086}, "yolo11_640_fire_11class": {"avg_ms": 47.576, "count": 100, "max_ms": 71.987, "p95_ms": 51.637}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.548, "count": 100, "max_ms": 77.473, "p95_ms": 53.279}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.635, "count": 101, "max_ms": 150.379, "p95_ms": 123.551}, "read_interval": {"avg_ms": 297.443, "count": 101, "max_ms": 19438.725, "p95_ms": 137.293}, "triton_wait": {"avg_ms": 55.17, "count": 100, "max_ms": 79.893, "p95_ms": 60.104}, "visualize": {"avg_ms": 3.688, "count": 100, "max_ms": 4.929, "p95_ms": 4.672}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10569.188, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:13:15+09:00", "last_error_summary": "[h264 @ 0x55d782d55c00] non-existing PPS 3 referenced [h264 @ 0x55d782d55c00] decode_slice_header error [h264 @ 0x55d783160480] cabac decode of qscale diff failed at 41 41 [h264 @ 0x55d783160480] error while decoding MB 41 41, bytestream 30590 [h264 @ 0x55d7824dc340] cabac decode of qscale diff failed at 29 21 [h264 @ 0x55d7824dc340] error while decoding MB 29 21, bytestream 5000 [h264 @ 0x55d782421e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d782421e00] error while dec", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 384, "h264_decode_error_count": 1259, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:13:15+09:00", "last_error_summary": "[h264 @ 0x55d782d55c00] non-existing PPS 3 referenced [h264 @ 0x55d782d55c00] decode_slice_header error [h264 @ 0x55d783160480] cabac decode of qscale diff failed at 41 41 [h264 @ 0x55d783160480] error while decoding MB 41 41, bytestream 30590 [h264 @ 0x55d7824dc340] cabac decode of qscale diff failed at 29 21 [h264 @ 0x55d7824dc340] error while decoding MB 29 21, bytestream 5000 [h264 @ 0x55d782421e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d782421e00] error while dec", "last_exit_at": "2026-07-01T05:13:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 265.069, "last_started_at": "2026-07-01T05:08:50+09:00", "pid": 80211, "process": {"exists": false, "pid": 80211}, "restart_count": 418, "restart_count_per_hour": 9, "rtsp_timeout_count": 378, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 419}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.718, "sample_count": 2182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1329907, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:13:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T05:13:15+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-01T05:13:15+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-01T05:13:15.062032+09:00", "mtime_epoch": 1782850395.0620325, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 142255}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:13:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4999.547, "name": "celery", "pid": 267, "rss_mb": 4999.547, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.401, "pushed": 3.401, "read": 3.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.392, "count": 101, "max_ms": 25.301, "p95_ms": 22.605}, "gstreamer_push": {"avg_ms": 1.188, "count": 102, "max_ms": 1.639, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.362, "count": 101, "max_ms": 55.615, "p95_ms": 32.84}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.156, "count": 101, "max_ms": 79.881, "p95_ms": 60.086}, "yolo11_640_fire_11class": {"avg_ms": 47.582, "count": 101, "max_ms": 71.987, "p95_ms": 51.637}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.56, "count": 101, "max_ms": 77.473, "p95_ms": 53.279}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.591, "count": 102, "max_ms": 150.379, "p95_ms": 123.551}, "read_interval": {"avg_ms": 297.443, "count": 101, "max_ms": 19438.725, "p95_ms": 137.293}, "triton_wait": {"avg_ms": 55.176, "count": 101, "max_ms": 79.893, "p95_ms": 60.104}, "visualize": {"avg_ms": 3.696, "count": 101, "max_ms": 4.929, "p95_ms": 4.672}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10569.008, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 384, "h264_decode_error_count": 1259, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:13:15+09:00", "last_error_summary": "[h264 @ 0x55d782d55c00] non-existing PPS 3 referenced [h264 @ 0x55d782d55c00] decode_slice_header error [h264 @ 0x55d783160480] cabac decode of qscale diff failed at 41 41 [h264 @ 0x55d783160480] error while decoding MB 41 41, bytestream 30590 [h264 @ 0x55d7824dc340] cabac decode of qscale diff failed at 29 21 [h264 @ 0x55d7824dc340] error while decoding MB 29 21, bytestream 5000 [h264 @ 0x55d782421e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d782421e00] error while dec", "last_exit_at": "2026-07-01T05:13:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 265.077, "last_started_at": "2026-07-01T05:08:50+09:00", "pid": 80211, "process": {"exists": false, "pid": 80211}, "restart_count": 418, "restart_count_per_hour": 9, "rtsp_timeout_count": 378, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 419}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.718, "sample_count": 2182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1329907, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:13:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-01T05:13:15+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-01T05:13:15+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-01T05:13:15.062032+09:00", "mtime_epoch": 1782850395.0620325, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 142255}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:13:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 4999.547, "name": "celery", "pid": 267, "rss_mb": 4999.547, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.4, "pushed": 3.4, "read": 3.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.392, "count": 101, "max_ms": 25.301, "p95_ms": 22.605}, "gstreamer_push": {"avg_ms": 1.188, "count": 102, "max_ms": 1.639, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.362, "count": 101, "max_ms": 55.615, "p95_ms": 32.84}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.156, "count": 101, "max_ms": 79.881, "p95_ms": 60.086}, "yolo11_640_fire_11class": {"avg_ms": 47.582, "count": 101, "max_ms": 71.987, "p95_ms": 51.637}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.56, "count": 101, "max_ms": 77.473, "p95_ms": 53.279}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.591, "count": 102, "max_ms": 150.379, "p95_ms": 123.551}, "read_interval": {"avg_ms": 297.443, "count": 101, "max_ms": 19438.725, "p95_ms": 137.293}, "triton_wait": {"avg_ms": 55.176, "count": 101, "max_ms": 79.893, "p95_ms": 60.104}, "visualize": {"avg_ms": 3.696, "count": 101, "max_ms": 4.929, "p95_ms": 4.672}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12218}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10568.645, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 193, "h264_decode_error_count": 198, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:04:17+09:00", "last_error_summary": "[h264 @ 0x55e195e11800] error while decoding MB 72 35, bytestream -7 [in#0/rtsp @ 0x55e19512fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e19512fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:04:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 535.58, "last_started_at": "2026-07-01T05:04:20+09:00", "pid": 79952, "process": {"exists": true, "fd_count": 5, "hwm_mb": 331.797, "name": "ffmpeg", "pid": 79952, "rss_mb": 331.797, "state": "S (sleeping)", "threads": 106}, "restart_count": 193, "restart_count_per_hour": 12, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 194}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.967, "sample_count": 2651, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1466257, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:12:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.059, "last_frame_pushed_at": "2026-07-01T05:12:46+09:00", "last_frame_read_age_sec": 30.136, "last_frame_read_at": "2026-07-01T05:12:46+09:00", "mjpeg": {"age_sec": 30.044, "exists": true, "mtime": "2026-07-01T05:12:46.201458+09:00", "mtime_epoch": 1782850366.2014577, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 160873}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:13:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1753.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12218}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10564.668, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 193, "h264_decode_error_count": 198, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:04:17+09:00", "last_error_summary": "[h264 @ 0x55e195e11800] error while decoding MB 72 35, bytestream -7 [in#0/rtsp @ 0x55e19512fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e19512fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:04:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 535.593, "last_started_at": "2026-07-01T05:04:20+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 193, "restart_count_per_hour": 12, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 194}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.967, "sample_count": 2651, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1466257, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:12:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.071, "last_frame_pushed_at": "2026-07-01T05:12:46+09:00", "last_frame_read_age_sec": 30.148, "last_frame_read_at": "2026-07-01T05:12:46+09:00", "mjpeg": {"age_sec": 30.057, "exists": true, "mtime": "2026-07-01T05:12:46.201458+09:00", "mtime_epoch": 1782850366.2014577, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 160873}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:13:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1753.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12218}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10266.785, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5651e46783c0] error while decoding MB 16 41, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 194, "h264_decode_error_count": 199, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:13:16+09:00", "last_error_summary": "[h264 @ 0x5651e46783c0] error while decoding MB 16 41, bytestream -9", "last_exit_at": "2026-07-01T05:04:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 535.644, "last_started_at": "2026-07-01T05:04:20+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 193, "restart_count_per_hour": 12, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 194}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.967, "sample_count": 2650, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1466257, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:12:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 30.122, "last_frame_pushed_at": "2026-07-01T05:12:46+09:00", "last_frame_read_age_sec": 30.2, "last_frame_read_at": "2026-07-01T05:12:46+09:00", "mjpeg": {"age_sec": 30.108, "exists": true, "mtime": "2026-07-01T05:12:46.201458+09:00", "mtime_epoch": 1782850366.2014577, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 160873}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:13:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1753.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10396.273, "limit_mb": 8796093022207.996}, "detail": {"pid": 80318, "restart_count": 419, "start_count": 420}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 384, "h264_decode_error_count": 1259, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:13:15+09:00", "last_error_summary": "[h264 @ 0x55d782d55c00] non-existing PPS 3 referenced [h264 @ 0x55d782d55c00] decode_slice_header error [h264 @ 0x55d783160480] cabac decode of qscale diff failed at 41 41 [h264 @ 0x55d783160480] error while decoding MB 41 41, bytestream 30590 [h264 @ 0x55d7824dc340] cabac decode of qscale diff failed at 29 21 [h264 @ 0x55d7824dc340] error while decoding MB 29 21, bytestream 5000 [h264 @ 0x55d782421e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d782421e00] error while dec", "last_exit_at": "2026-07-01T05:13:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:13:17+09:00", "pid": 80318, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.422, "name": "ffmpeg", "pid": 80318, "rss_mb": 139.422, "state": "S (sleeping)", "threads": 106}, "restart_count": 419, "restart_count_per_hour": 10, "rtsp_timeout_count": 378, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 420}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.718, "sample_count": 2154, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1329907, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:13:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.502, "last_frame_pushed_at": "2026-07-01T05:13:15+09:00", "last_frame_read_age_sec": 4.592, "last_frame_read_at": "2026-07-01T05:13:15+09:00", "mjpeg": {"age_sec": 4.486, "exists": true, "mtime": "2026-07-01T05:13:15.162034+09:00", "mtime_epoch": 1782850395.1620343, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144244}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:13:19+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 4999.547, "name": "celery", "pid": 267, "rss_mb": 4999.547, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.167, "pushed": 2.167, "read": 2.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.844, "count": 65, "max_ms": 22.605, "p95_ms": 20.711}, "gstreamer_push": {"avg_ms": 1.21, "count": 65, "max_ms": 1.365, "p95_ms": 1.317}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.577, "count": 65, "max_ms": 55.615, "p95_ms": 33.468}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.309, "count": 65, "max_ms": 79.881, "p95_ms": 55.744}, "yolo11_640_fire_11class": {"avg_ms": 47.018, "count": 65, "max_ms": 71.987, "p95_ms": 48.603}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.588, "count": 65, "max_ms": 73.181, "p95_ms": 49.755}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.147, "count": 65, "max_ms": 107.721, "p95_ms": 83.679}, "read_interval": {"avg_ms": 398.126, "count": 64, "max_ms": 19438.725, "p95_ms": 97.912}, "triton_wait": {"avg_ms": 54.321, "count": 65, "max_ms": 79.893, "p95_ms": 55.756}, "visualize": {"avg_ms": 3.405, "count": 65, "max_ms": 4.458, "p95_ms": 3.59}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12218}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10404.316, "limit_mb": 8796093022207.996}, "detail": {"pid": 80320, "restart_count": 194, "start_count": 195}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 194, "h264_decode_error_count": 199, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:13:16+09:00", "last_error_summary": "[h264 @ 0x5651e46783c0] error while decoding MB 16 41, bytestream -9", "last_exit_at": "2026-07-01T05:04:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:13:18+09:00", "pid": 80320, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.352, "name": "ffmpeg", "pid": 80320, "rss_mb": 58.352, "state": "S (sleeping)", "threads": 1}, "restart_count": 194, "restart_count_per_hour": 13, "rtsp_timeout_count": 316, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 195}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.954, "sample_count": 2607, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1466257, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:12:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 34.548, "last_frame_pushed_at": "2026-07-01T05:12:46+09:00", "last_frame_read_age_sec": 34.625, "last_frame_read_at": "2026-07-01T05:12:46+09:00", "mjpeg": {"age_sec": 34.534, "exists": true, "mtime": "2026-07-01T05:12:46.201458+09:00", "mtime_epoch": 1782850366.2014577, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 160873}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:13:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1753.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10573.414, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a132ca1840] left block unavailable for requested intra mode [h264 @ 0x55a132ca1840] error while decoding MB 0 37, bytestream 2591 [h264 @ 0x55a133a2a000] error while decoding MB 76 37, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 385, "h264_decode_error_count": 1261, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:18:36+09:00", "last_error_summary": "[h264 @ 0x55a132ca1840] left block unavailable for requested intra mode [h264 @ 0x55a132ca1840] error while decoding MB 0 37, bytestream 2591 [h264 @ 0x55a133a2a000] error while decoding MB 76 37, bytestream -5", "last_exit_at": "2026-07-01T05:13:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 319.319, "last_started_at": "2026-07-01T05:13:17+09:00", "pid": 80318, "process": {"exists": false, "pid": 80318}, "restart_count": 419, "restart_count_per_hour": 9, "rtsp_timeout_count": 378, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 420}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.567, "sample_count": 2511, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1332579, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:18:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-01T05:18:36+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T05:18:36+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-01T05:18:36.914769+09:00", "mtime_epoch": 1782850716.914769, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144827}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:18:36+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5000.836, "name": "celery", "pid": 267, "rss_mb": 5000.836, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 252, "pushed_frames": 252, "read_frames": 252, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.418, "pushed": 8.418, "read": 8.418}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.19, "count": 252, "max_ms": 23.02, "p95_ms": 21.185}, "gstreamer_push": {"avg_ms": 1.215, "count": 252, "max_ms": 1.338, "p95_ms": 1.294}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.296, "count": 252, "max_ms": 45.032, "p95_ms": 42.106}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.714, "count": 252, "max_ms": 84.284, "p95_ms": 80.158}, "yolo11_640_fire_11class": {"avg_ms": 60.249, "count": 252, "max_ms": 67.333, "p95_ms": 65.698}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.435, "count": 252, "max_ms": 80.08, "p95_ms": 77.457}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.92, "count": 252, "max_ms": 304.048, "p95_ms": 108.314}, "read_interval": {"avg_ms": 118.982, "count": 252, "max_ms": 314.367, "p95_ms": 124.099}, "triton_wait": {"avg_ms": 74.747, "count": 252, "max_ms": 84.294, "p95_ms": 80.17}, "visualize": {"avg_ms": 3.406, "count": 252, "max_ms": 4.752, "p95_ms": 3.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10572.754, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:18:36+09:00", "last_error_summary": "[h264 @ 0x55a132ca1840] left block unavailable for requested intra mode [h264 @ 0x55a132ca1840] error while decoding MB 0 37, bytestream 2591 [h264 @ 0x55a133a2a000] error while decoding MB 76 37, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 385, "h264_decode_error_count": 1261, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:18:36+09:00", "last_error_summary": "[h264 @ 0x55a132ca1840] left block unavailable for requested intra mode [h264 @ 0x55a132ca1840] error while decoding MB 0 37, bytestream 2591 [h264 @ 0x55a133a2a000] error while decoding MB 76 37, bytestream -5", "last_exit_at": "2026-07-01T05:18:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 319.379, "last_started_at": "2026-07-01T05:13:17+09:00", "pid": 80318, "process": {"exists": false, "pid": 80318}, "restart_count": 419, "restart_count_per_hour": 9, "rtsp_timeout_count": 378, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 420}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.567, "sample_count": 2512, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1332580, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:18:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T05:18:37+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-01T05:18:36+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-01T05:18:36.914769+09:00", "mtime_epoch": 1782850716.914769, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144827}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:18:37+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5000.836, "name": "celery", "pid": 267, "rss_mb": 5000.836, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 253, "pushed_frames": 253, "read_frames": 252, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.435, "pushed": 8.435, "read": 8.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.188, "count": 253, "max_ms": 23.02, "p95_ms": 21.185}, "gstreamer_push": {"avg_ms": 1.215, "count": 253, "max_ms": 1.338, "p95_ms": 1.294}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.286, "count": 253, "max_ms": 45.032, "p95_ms": 42.106}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.728, "count": 253, "max_ms": 84.284, "p95_ms": 80.158}, "yolo11_640_fire_11class": {"avg_ms": 60.266, "count": 253, "max_ms": 67.333, "p95_ms": 65.698}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.457, "count": 253, "max_ms": 80.08, "p95_ms": 77.457}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.927, "count": 253, "max_ms": 304.048, "p95_ms": 108.314}, "read_interval": {"avg_ms": 118.982, "count": 252, "max_ms": 314.367, "p95_ms": 124.099}, "triton_wait": {"avg_ms": 74.76, "count": 253, "max_ms": 84.294, "p95_ms": 80.17}, "visualize": {"avg_ms": 3.405, "count": 253, "max_ms": 4.752, "p95_ms": 3.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10572.574, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 385, "h264_decode_error_count": 1261, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:18:36+09:00", "last_error_summary": "[h264 @ 0x55a132ca1840] left block unavailable for requested intra mode [h264 @ 0x55a132ca1840] error while decoding MB 0 37, bytestream 2591 [h264 @ 0x55a133a2a000] error while decoding MB 76 37, bytestream -5", "last_exit_at": "2026-07-01T05:18:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 319.386, "last_started_at": "2026-07-01T05:13:17+09:00", "pid": 80318, "process": {"exists": false, "pid": 80318}, "restart_count": 419, "restart_count_per_hour": 9, "rtsp_timeout_count": 378, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 420}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.567, "sample_count": 2512, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1332580, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:18:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T05:18:37+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-01T05:18:36+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-01T05:18:36.914769+09:00", "mtime_epoch": 1782850716.914769, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144827}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:18:37+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5000.836, "name": "celery", "pid": 267, "rss_mb": 5000.836, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 252, "pushed_frames": 253, "read_frames": 252, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.4, "pushed": 8.434, "read": 8.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.19, "count": 252, "max_ms": 23.02, "p95_ms": 21.185}, "gstreamer_push": {"avg_ms": 1.215, "count": 253, "max_ms": 1.338, "p95_ms": 1.294}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.258, "count": 252, "max_ms": 45.032, "p95_ms": 42.106}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.721, "count": 252, "max_ms": 84.284, "p95_ms": 80.158}, "yolo11_640_fire_11class": {"avg_ms": 60.259, "count": 252, "max_ms": 67.333, "p95_ms": 65.698}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.439, "count": 252, "max_ms": 80.08, "p95_ms": 77.457}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.925, "count": 252, "max_ms": 304.048, "p95_ms": 108.314}, "read_interval": {"avg_ms": 118.982, "count": 252, "max_ms": 314.367, "p95_ms": 124.099}, "triton_wait": {"avg_ms": 74.754, "count": 252, "max_ms": 84.294, "p95_ms": 80.17}, "visualize": {"avg_ms": 3.405, "count": 252, "max_ms": 4.752, "p95_ms": 3.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10681.77, "limit_mb": 8796093022207.996}, "detail": {"pid": 80577, "restart_count": 420, "start_count": 421}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 385, "h264_decode_error_count": 1261, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:18:36+09:00", "last_error_summary": "[h264 @ 0x55a132ca1840] left block unavailable for requested intra mode [h264 @ 0x55a132ca1840] error while decoding MB 0 37, bytestream 2591 [h264 @ 0x55a133a2a000] error while decoding MB 76 37, bytestream -5", "last_exit_at": "2026-07-01T05:18:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:18:39+09:00", "pid": 80577, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.191, "name": "ffmpeg", "pid": 80577, "rss_mb": 139.191, "state": "S (sleeping)", "threads": 106}, "restart_count": 420, "restart_count_per_hour": 10, "rtsp_timeout_count": 378, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 421}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.602, "sample_count": 2470, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1332580, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:18:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.557, "last_frame_pushed_at": "2026-07-01T05:18:37+09:00", "last_frame_read_age_sec": 4.665, "last_frame_read_at": "2026-07-01T05:18:36+09:00", "mjpeg": {"age_sec": 4.542, "exists": true, "mtime": "2026-07-01T05:18:37.034772+09:00", "mtime_epoch": 1782850717.0347722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144529}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:18:41+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5000.836, "name": "celery", "pid": 267, "rss_mb": 5000.836, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.18, "pushed": 7.18, "read": 7.146}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.186, "count": 215, "max_ms": 23.02, "p95_ms": 21.337}, "gstreamer_push": {"avg_ms": 1.211, "count": 215, "max_ms": 1.338, "p95_ms": 1.287}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.791, "count": 215, "max_ms": 45.032, "p95_ms": 41.862}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.247, "count": 215, "max_ms": 84.284, "p95_ms": 80.115}, "yolo11_640_fire_11class": {"avg_ms": 59.816, "count": 215, "max_ms": 67.333, "p95_ms": 65.464}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.926, "count": 215, "max_ms": 80.08, "p95_ms": 77.626}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.57, "count": 215, "max_ms": 304.048, "p95_ms": 108.192}, "read_interval": {"avg_ms": 118.627, "count": 214, "max_ms": 314.367, "p95_ms": 124.081}, "triton_wait": {"avg_ms": 74.284, "count": 215, "max_ms": 84.294, "p95_ms": 80.128}, "visualize": {"avg_ms": 3.4, "count": 215, "max_ms": 4.752, "p95_ms": 3.559}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11120}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10577.02, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557b912afbc0] cabac decode of qscale diff failed at 33 40 [h264 @ 0x557b912afbc0] error while decoding MB 33 40, bytestream 1946 [h264 @ 0x557b912ec040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b912ec040] error while decoding MB 0 34, bytestream 1501 [h264 @ 0x557b91e6f8c0] left block unavailable for requested intra mode [h264 @ 0x557b91e6f8c0] error while decoding MB 0 3, bytestream 77129 [h264 @ 0x557b912d7e80] error while decoding MB 42 14, bytestream -7 [i", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 386, "h264_decode_error_count": 1266, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:23:17+09:00", "last_error_summary": "[h264 @ 0x557b912afbc0] cabac decode of qscale diff failed at 33 40 [h264 @ 0x557b912afbc0] error while decoding MB 33 40, bytestream 1946 [h264 @ 0x557b912ec040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b912ec040] error while decoding MB 0 34, bytestream 1501 [h264 @ 0x557b91e6f8c0] left block unavailable for requested intra mode [h264 @ 0x557b91e6f8c0] error while decoding MB 0 3, bytestream 77129 [h264 @ 0x557b912d7e80] error while decoding MB 42 14, bytestream -7 [i", "last_exit_at": "2026-07-01T05:18:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.887, "last_started_at": "2026-07-01T05:18:39+09:00", "pid": 80577, "process": {"exists": false, "pid": 80577}, "restart_count": 420, "restart_count_per_hour": 10, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 421}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.173, "sample_count": 2017, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1334432, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:23:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T05:23:17+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T05:23:17+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T05:23:17.418174+09:00", "mtime_epoch": 1782850997.4181738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:23:17+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5001.609, "name": "celery", "pid": 267, "rss_mb": 5001.609, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.776, "pushed": 2.776, "read": 2.776}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.948, "count": 83, "max_ms": 27.79, "p95_ms": 26.35}, "gstreamer_push": {"avg_ms": 1.135, "count": 83, "max_ms": 1.648, "p95_ms": 1.235}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.313, "count": 83, "max_ms": 39.94, "p95_ms": 35.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.566, "count": 83, "max_ms": 84.139, "p95_ms": 79.196}, "yolo11_640_fire_11class": {"avg_ms": 49.122, "count": 83, "max_ms": 65.419, "p95_ms": 62.3}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.421, "count": 83, "max_ms": 82.367, "p95_ms": 77.522}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.397, "count": 83, "max_ms": 150.287, "p95_ms": 146.286}, "read_interval": {"avg_ms": 361.072, "count": 83, "max_ms": 10124.258, "p95_ms": 158.881}, "triton_wait": {"avg_ms": 61.881, "count": 83, "max_ms": 84.151, "p95_ms": 79.209}, "visualize": {"avg_ms": 4.658, "count": 83, "max_ms": 6.36, "p95_ms": 5.582}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11120}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10576.797, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:23:17+09:00", "last_error_summary": "[h264 @ 0x557b912afbc0] cabac decode of qscale diff failed at 33 40 [h264 @ 0x557b912afbc0] error while decoding MB 33 40, bytestream 1946 [h264 @ 0x557b912ec040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b912ec040] error while decoding MB 0 34, bytestream 1501 [h264 @ 0x557b91e6f8c0] left block unavailable for requested intra mode [h264 @ 0x557b91e6f8c0] error while decoding MB 0 3, bytestream 77129 [h264 @ 0x557b912d7e80] error while decoding MB 42 14, bytestream -7 [i", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 386, "h264_decode_error_count": 1266, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:23:17+09:00", "last_error_summary": "[h264 @ 0x557b912afbc0] cabac decode of qscale diff failed at 33 40 [h264 @ 0x557b912afbc0] error while decoding MB 33 40, bytestream 1946 [h264 @ 0x557b912ec040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b912ec040] error while decoding MB 0 34, bytestream 1501 [h264 @ 0x557b91e6f8c0] left block unavailable for requested intra mode [h264 @ 0x557b91e6f8c0] error while decoding MB 0 3, bytestream 77129 [h264 @ 0x557b912d7e80] error while decoding MB 42 14, bytestream -7 [i", "last_exit_at": "2026-07-01T05:23:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.969, "last_started_at": "2026-07-01T05:18:39+09:00", "pid": 80577, "process": {"exists": false, "pid": 80577}, "restart_count": 420, "restart_count_per_hour": 10, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 421}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.173, "sample_count": 2018, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1334433, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:23:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T05:23:17+09:00", "last_frame_read_age_sec": 0.135, "last_frame_read_at": "2026-07-01T05:23:17+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-01T05:23:17.418174+09:00", "mtime_epoch": 1782850997.4181738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:23:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5001.609, "name": "celery", "pid": 267, "rss_mb": 5001.609, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 83, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.802, "pushed": 2.802, "read": 2.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.927, "count": 83, "max_ms": 27.79, "p95_ms": 26.35}, "gstreamer_push": {"avg_ms": 1.136, "count": 84, "max_ms": 1.648, "p95_ms": 1.235}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.319, "count": 83, "max_ms": 39.94, "p95_ms": 35.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.548, "count": 83, "max_ms": 84.139, "p95_ms": 79.196}, "yolo11_640_fire_11class": {"avg_ms": 49.08, "count": 83, "max_ms": 65.419, "p95_ms": 62.3}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.384, "count": 83, "max_ms": 82.367, "p95_ms": 77.522}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.425, "count": 84, "max_ms": 150.287, "p95_ms": 146.286}, "read_interval": {"avg_ms": 361.072, "count": 83, "max_ms": 10124.258, "p95_ms": 158.881}, "triton_wait": {"avg_ms": 61.863, "count": 83, "max_ms": 84.151, "p95_ms": 79.209}, "visualize": {"avg_ms": 4.676, "count": 83, "max_ms": 6.36, "p95_ms": 5.665}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11120}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10576.785, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 386, "h264_decode_error_count": 1266, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:23:17+09:00", "last_error_summary": "[h264 @ 0x557b912afbc0] cabac decode of qscale diff failed at 33 40 [h264 @ 0x557b912afbc0] error while decoding MB 33 40, bytestream 1946 [h264 @ 0x557b912ec040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b912ec040] error while decoding MB 0 34, bytestream 1501 [h264 @ 0x557b91e6f8c0] left block unavailable for requested intra mode [h264 @ 0x557b91e6f8c0] error while decoding MB 0 3, bytestream 77129 [h264 @ 0x557b912d7e80] error while decoding MB 42 14, bytestream -7 [i", "last_exit_at": "2026-07-01T05:23:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.974, "last_started_at": "2026-07-01T05:18:39+09:00", "pid": 80577, "process": {"exists": false, "pid": 80577}, "restart_count": 420, "restart_count_per_hour": 10, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 421}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.173, "sample_count": 2018, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1334433, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:23:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T05:23:17+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-01T05:23:17+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-01T05:23:17.418174+09:00", "mtime_epoch": 1782850997.4181738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:23:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5001.609, "name": "celery", "pid": 267, "rss_mb": 5001.609, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 83, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.801, "pushed": 2.801, "read": 2.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.927, "count": 83, "max_ms": 27.79, "p95_ms": 26.35}, "gstreamer_push": {"avg_ms": 1.136, "count": 84, "max_ms": 1.648, "p95_ms": 1.235}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.319, "count": 83, "max_ms": 39.94, "p95_ms": 35.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.548, "count": 83, "max_ms": 84.139, "p95_ms": 79.196}, "yolo11_640_fire_11class": {"avg_ms": 49.08, "count": 83, "max_ms": 65.419, "p95_ms": 62.3}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.384, "count": 83, "max_ms": 82.367, "p95_ms": 77.522}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.425, "count": 84, "max_ms": 150.287, "p95_ms": 146.286}, "read_interval": {"avg_ms": 361.072, "count": 83, "max_ms": 10124.258, "p95_ms": 158.881}, "triton_wait": {"avg_ms": 61.863, "count": 83, "max_ms": 84.151, "p95_ms": 79.209}, "visualize": {"avg_ms": 4.676, "count": 83, "max_ms": 6.36, "p95_ms": 5.665}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11120}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10684.379, "limit_mb": 8796093022207.996}, "detail": {"pid": 80684, "restart_count": 421, "start_count": 422}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 386, "h264_decode_error_count": 1266, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:23:17+09:00", "last_error_summary": "[h264 @ 0x557b912afbc0] cabac decode of qscale diff failed at 33 40 [h264 @ 0x557b912afbc0] error while decoding MB 33 40, bytestream 1946 [h264 @ 0x557b912ec040] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b912ec040] error while decoding MB 0 34, bytestream 1501 [h264 @ 0x557b91e6f8c0] left block unavailable for requested intra mode [h264 @ 0x557b91e6f8c0] error while decoding MB 0 3, bytestream 77129 [h264 @ 0x557b912d7e80] error while decoding MB 42 14, bytestream -7 [i", "last_exit_at": "2026-07-01T05:23:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:23:19+09:00", "pid": 80684, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.496, "name": "ffmpeg", "pid": 80684, "rss_mb": 138.496, "state": "S (sleeping)", "threads": 106}, "restart_count": 421, "restart_count_per_hour": 11, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 422}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.184, "sample_count": 1982, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1334433, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:23:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.152, "last_frame_pushed_at": "2026-07-01T05:23:17+09:00", "last_frame_read_age_sec": 4.282, "last_frame_read_at": "2026-07-01T05:23:17+09:00", "mjpeg": {"age_sec": 4.134, "exists": true, "mtime": "2026-07-01T05:23:17.558177+09:00", "mtime_epoch": 1782850997.5581772, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167659}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:23:21+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5001.609, "name": "celery", "pid": 267, "rss_mb": 5001.609, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 52, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.773, "pushed": 1.773, "read": 1.74}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.76, "count": 53, "max_ms": 27.79, "p95_ms": 25.941}, "gstreamer_push": {"avg_ms": 1.128, "count": 53, "max_ms": 1.648, "p95_ms": 1.235}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.404, "count": 53, "max_ms": 38.425, "p95_ms": 35.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.94, "count": 53, "max_ms": 84.139, "p95_ms": 82.981}, "yolo11_640_fire_11class": {"avg_ms": 49.354, "count": 53, "max_ms": 65.419, "p95_ms": 64.453}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.839, "count": 53, "max_ms": 82.367, "p95_ms": 79.823}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.618, "count": 53, "max_ms": 150.287, "p95_ms": 147.822}, "read_interval": {"avg_ms": 494.928, "count": 52, "max_ms": 10124.258, "p95_ms": 160.5}, "triton_wait": {"avg_ms": 62.153, "count": 53, "max_ms": 84.151, "p95_ms": 82.993}, "visualize": {"avg_ms": 4.692, "count": 53, "max_ms": 6.36, "p95_ms": 5.715}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.886, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10579.906, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5615ba9968c0] error while decoding MB 65 17, bytestream -16", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 387, "h264_decode_error_count": 1267, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:26:30+09:00", "last_error_summary": "[h264 @ 0x5615ba9968c0] error while decoding MB 65 17, bytestream -16", "last_exit_at": "2026-07-01T05:23:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.831, "last_started_at": "2026-07-01T05:23:19+09:00", "pid": 80684, "process": {"exists": false, "pid": 80684}, "restart_count": 421, "restart_count_per_hour": 11, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 422}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.12, "sample_count": 2002, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1335805, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:26:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T05:26:30+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T05:26:30+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T05:26:30.474869+09:00", "mtime_epoch": 1782851190.4748693, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165049}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:26:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5002.383, "name": "celery", "pid": 267, "rss_mb": 5002.383, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 218, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.287, "pushed": 7.287, "read": 7.287}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.683, "count": 218, "max_ms": 29.323, "p95_ms": 25.285}, "gstreamer_push": {"avg_ms": 1.113, "count": 218, "max_ms": 1.544, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.971, "count": 218, "max_ms": 41.35, "p95_ms": 37.138}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.269, "count": 218, "max_ms": 89.047, "p95_ms": 80.166}, "yolo11_640_fire_11class": {"avg_ms": 48.914, "count": 218, "max_ms": 72.299, "p95_ms": 62.941}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.945, "count": 218, "max_ms": 80.938, "p95_ms": 77.088}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.222, "count": 218, "max_ms": 152.112, "p95_ms": 143.359}, "read_interval": {"avg_ms": 137.526, "count": 218, "max_ms": 163.96, "p95_ms": 155.175}, "triton_wait": {"avg_ms": 61.416, "count": 218, "max_ms": 89.06, "p95_ms": 80.178}, "visualize": {"avg_ms": 4.591, "count": 218, "max_ms": 5.895, "p95_ms": 4.872}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10579.176, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:26:30+09:00", "last_error_summary": "[h264 @ 0x5615ba9968c0] error while decoding MB 65 17, bytestream -16", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 387, "h264_decode_error_count": 1267, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:26:30+09:00", "last_error_summary": "[h264 @ 0x5615ba9968c0] error while decoding MB 65 17, bytestream -16", "last_exit_at": "2026-07-01T05:26:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.926, "last_started_at": "2026-07-01T05:23:19+09:00", "pid": 80684, "process": {"exists": false, "pid": 80684}, "restart_count": 421, "restart_count_per_hour": 11, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 422}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.12, "sample_count": 2002, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1335806, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:26:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T05:26:30+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-01T05:26:30+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-01T05:26:30.474869+09:00", "mtime_epoch": 1782851190.4748693, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165049}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:26:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5002.383, "name": "celery", "pid": 267, "rss_mb": 5002.383, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 218, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.267, "pushed": 7.267, "read": 7.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.681, "count": 218, "max_ms": 29.323, "p95_ms": 25.285}, "gstreamer_push": {"avg_ms": 1.113, "count": 218, "max_ms": 1.544, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.957, "count": 218, "max_ms": 41.35, "p95_ms": 37.138}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.361, "count": 218, "max_ms": 89.047, "p95_ms": 80.166}, "yolo11_640_fire_11class": {"avg_ms": 48.988, "count": 218, "max_ms": 72.299, "p95_ms": 62.941}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.072, "count": 218, "max_ms": 80.938, "p95_ms": 77.088}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.335, "count": 218, "max_ms": 152.112, "p95_ms": 143.359}, "read_interval": {"avg_ms": 137.526, "count": 218, "max_ms": 163.96, "p95_ms": 155.175}, "triton_wait": {"avg_ms": 61.516, "count": 218, "max_ms": 89.06, "p95_ms": 80.178}, "visualize": {"avg_ms": 4.59, "count": 218, "max_ms": 5.895, "p95_ms": 4.852}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10579.168, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 387, "h264_decode_error_count": 1267, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:26:30+09:00", "last_error_summary": "[h264 @ 0x5615ba9968c0] error while decoding MB 65 17, bytestream -16", "last_exit_at": "2026-07-01T05:26:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.932, "last_started_at": "2026-07-01T05:23:19+09:00", "pid": 80684, "process": {"exists": false, "pid": 80684}, "restart_count": 421, "restart_count_per_hour": 11, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 422}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.12, "sample_count": 2002, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1335806, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:26:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T05:26:30+09:00", "last_frame_read_age_sec": 0.153, "last_frame_read_at": "2026-07-01T05:26:30+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-01T05:26:30.474869+09:00", "mtime_epoch": 1782851190.4748693, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165049}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:26:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5002.383, "name": "celery", "pid": 267, "rss_mb": 5002.383, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 218, "pushed_frames": 218, "read_frames": 217, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.299, "pushed": 7.299, "read": 7.265}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.681, "count": 218, "max_ms": 29.323, "p95_ms": 25.285}, "gstreamer_push": {"avg_ms": 1.113, "count": 218, "max_ms": 1.544, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.957, "count": 218, "max_ms": 41.35, "p95_ms": 37.138}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.361, "count": 218, "max_ms": 89.047, "p95_ms": 80.166}, "yolo11_640_fire_11class": {"avg_ms": 48.988, "count": 218, "max_ms": 72.299, "p95_ms": 62.941}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.072, "count": 218, "max_ms": 80.938, "p95_ms": 77.088}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.335, "count": 218, "max_ms": 152.112, "p95_ms": 143.359}, "read_interval": {"avg_ms": 137.564, "count": 217, "max_ms": 163.96, "p95_ms": 155.175}, "triton_wait": {"avg_ms": 61.516, "count": 218, "max_ms": 89.06, "p95_ms": 80.178}, "visualize": {"avg_ms": 4.59, "count": 218, "max_ms": 5.895, "p95_ms": 4.852}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10689.488, "limit_mb": 8796093022207.996}, "detail": {"pid": 80791, "restart_count": 422, "start_count": 423}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 387, "h264_decode_error_count": 1267, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:26:30+09:00", "last_error_summary": "[h264 @ 0x5615ba9968c0] error while decoding MB 65 17, bytestream -16", "last_exit_at": "2026-07-01T05:26:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:26:33+09:00", "pid": 80791, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.496, "name": "ffmpeg", "pid": 80791, "rss_mb": 141.496, "state": "S (sleeping)", "threads": 106}, "restart_count": 422, "restart_count_per_hour": 12, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 423}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.12, "sample_count": 1966, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1335806, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:26:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.987, "last_frame_pushed_at": "2026-07-01T05:26:30+09:00", "last_frame_read_age_sec": 5.13, "last_frame_read_at": "2026-07-01T05:26:30+09:00", "mjpeg": {"age_sec": 4.969, "exists": true, "mtime": "2026-07-01T05:26:30.630873+09:00", "mtime_epoch": 1782851190.6308727, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163639}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:26:35+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5002.383, "name": "celery", "pid": 267, "rss_mb": 5002.383, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 182, "pushed_frames": 182, "read_frames": 181, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.092, "pushed": 6.092, "read": 6.058}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.75, "count": 182, "max_ms": 29.323, "p95_ms": 25.316}, "gstreamer_push": {"avg_ms": 1.114, "count": 182, "max_ms": 1.544, "p95_ms": 1.164}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.878, "count": 182, "max_ms": 41.35, "p95_ms": 36.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.127, "count": 182, "max_ms": 83.579, "p95_ms": 79.054}, "yolo11_640_fire_11class": {"avg_ms": 48.844, "count": 182, "max_ms": 66.106, "p95_ms": 62.666}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.904, "count": 182, "max_ms": 80.938, "p95_ms": 76.716}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.196, "count": 182, "max_ms": 148.305, "p95_ms": 142.401}, "read_interval": {"avg_ms": 137.41, "count": 181, "max_ms": 161.1, "p95_ms": 154.503}, "triton_wait": {"avg_ms": 61.304, "count": 182, "max_ms": 83.591, "p95_ms": 79.063}, "visualize": {"avg_ms": 4.585, "count": 182, "max_ms": 5.895, "p95_ms": 4.816}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10582.047, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5598f6c33480] error while decoding MB 31 8, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 388, "h264_decode_error_count": 1268, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:29:44+09:00", "last_error_summary": "[h264 @ 0x5598f6c33480] error while decoding MB 31 8, bytestream -6", "last_exit_at": "2026-07-01T05:26:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.279, "last_started_at": "2026-07-01T05:26:33+09:00", "pid": 80791, "process": {"exists": false, "pid": 80791}, "restart_count": 422, "restart_count_per_hour": 11, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 423}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.321, "sample_count": 2142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1337178, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:29:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T05:29:44+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T05:29:44+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T05:29:44.831384+09:00", "mtime_epoch": 1782851384.8313837, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158618}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:29:44+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5002.898, "name": "celery", "pid": 267, "rss_mb": 5002.898, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.226, "pushed": 7.226, "read": 7.226}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.642, "count": 216, "max_ms": 26.35, "p95_ms": 24.703}, "gstreamer_push": {"avg_ms": 1.12, "count": 216, "max_ms": 1.645, "p95_ms": 1.184}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.03, "count": 216, "max_ms": 43.005, "p95_ms": 34.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.988, "count": 216, "max_ms": 84.187, "p95_ms": 80.667}, "yolo11_640_fire_11class": {"avg_ms": 48.569, "count": 216, "max_ms": 65.007, "p95_ms": 61.029}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.188, "count": 216, "max_ms": 80.73, "p95_ms": 76.416}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.209, "count": 216, "max_ms": 149.844, "p95_ms": 143.634}, "read_interval": {"avg_ms": 138.695, "count": 216, "max_ms": 162.052, "p95_ms": 157.284}, "triton_wait": {"avg_ms": 62.097, "count": 216, "max_ms": 84.199, "p95_ms": 80.679}, "visualize": {"avg_ms": 4.612, "count": 216, "max_ms": 5.746, "p95_ms": 4.997}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10581.664, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:29:44+09:00", "last_error_summary": "[h264 @ 0x5598f6c33480] error while decoding MB 31 8, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 388, "h264_decode_error_count": 1268, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:29:44+09:00", "last_error_summary": "[h264 @ 0x5598f6c33480] error while decoding MB 31 8, bytestream -6", "last_exit_at": "2026-07-01T05:29:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.376, "last_started_at": "2026-07-01T05:26:33+09:00", "pid": 80791, "process": {"exists": false, "pid": 80791}, "restart_count": 422, "restart_count_per_hour": 11, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 423}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.321, "sample_count": 2142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1337179, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:29:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T05:29:44+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-01T05:29:44+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-01T05:29:44.831384+09:00", "mtime_epoch": 1782851384.8313837, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158618}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:29:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5002.898, "name": "celery", "pid": 267, "rss_mb": 5002.898, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.236, "pushed": 7.236, "read": 7.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.643, "count": 216, "max_ms": 26.35, "p95_ms": 24.703}, "gstreamer_push": {"avg_ms": 1.12, "count": 217, "max_ms": 1.645, "p95_ms": 1.19}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.031, "count": 216, "max_ms": 43.005, "p95_ms": 34.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.053, "count": 216, "max_ms": 84.187, "p95_ms": 80.667}, "yolo11_640_fire_11class": {"avg_ms": 48.673, "count": 216, "max_ms": 67.012, "p95_ms": 62.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.328, "count": 216, "max_ms": 80.73, "p95_ms": 76.559}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.28, "count": 217, "max_ms": 149.844, "p95_ms": 143.634}, "read_interval": {"avg_ms": 138.695, "count": 216, "max_ms": 162.052, "p95_ms": 157.284}, "triton_wait": {"avg_ms": 62.208, "count": 216, "max_ms": 84.199, "p95_ms": 80.679}, "visualize": {"avg_ms": 4.612, "count": 216, "max_ms": 5.746, "p95_ms": 4.997}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10581.656, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 388, "h264_decode_error_count": 1268, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:29:44+09:00", "last_error_summary": "[h264 @ 0x5598f6c33480] error while decoding MB 31 8, bytestream -6", "last_exit_at": "2026-07-01T05:29:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.382, "last_started_at": "2026-07-01T05:26:33+09:00", "pid": 80791, "process": {"exists": false, "pid": 80791}, "restart_count": 422, "restart_count_per_hour": 11, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 423}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.321, "sample_count": 2142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1337179, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:29:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T05:29:44+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-01T05:29:44+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-01T05:29:44.831384+09:00", "mtime_epoch": 1782851384.8313837, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158618}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:29:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5002.898, "name": "celery", "pid": 267, "rss_mb": 5002.898, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.235, "pushed": 7.235, "read": 7.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.643, "count": 216, "max_ms": 26.35, "p95_ms": 24.703}, "gstreamer_push": {"avg_ms": 1.12, "count": 217, "max_ms": 1.645, "p95_ms": 1.19}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.031, "count": 216, "max_ms": 43.005, "p95_ms": 34.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.053, "count": 216, "max_ms": 84.187, "p95_ms": 80.667}, "yolo11_640_fire_11class": {"avg_ms": 48.673, "count": 216, "max_ms": 67.012, "p95_ms": 62.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.328, "count": 216, "max_ms": 80.73, "p95_ms": 76.559}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.28, "count": 217, "max_ms": 149.844, "p95_ms": 143.634}, "read_interval": {"avg_ms": 138.695, "count": 216, "max_ms": 162.052, "p95_ms": 157.284}, "triton_wait": {"avg_ms": 62.208, "count": 216, "max_ms": 84.199, "p95_ms": 80.679}, "visualize": {"avg_ms": 4.612, "count": 216, "max_ms": 5.746, "p95_ms": 4.997}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10689.094, "limit_mb": 8796093022207.996}, "detail": {"pid": 80898, "restart_count": 423, "start_count": 424}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 388, "h264_decode_error_count": 1268, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:29:44+09:00", "last_error_summary": "[h264 @ 0x5598f6c33480] error while decoding MB 31 8, bytestream -6", "last_exit_at": "2026-07-01T05:29:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:29:47+09:00", "pid": 80898, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.957, "name": "ffmpeg", "pid": 80898, "rss_mb": 138.957, "state": "R (running)", "threads": 106}, "restart_count": 423, "restart_count_per_hour": 12, "rtsp_timeout_count": 380, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 424}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.33, "sample_count": 2108, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1337179, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:29:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.496, "last_frame_pushed_at": "2026-07-01T05:29:44+09:00", "last_frame_read_age_sec": 4.639, "last_frame_read_at": "2026-07-01T05:29:44+09:00", "mjpeg": {"age_sec": 4.478, "exists": true, "mtime": "2026-07-01T05:29:44.987387+09:00", "mtime_epoch": 1782851384.9873872, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158536}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:29:49+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5002.898, "name": "celery", "pid": 267, "rss_mb": 5002.898, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.155, "pushed": 6.155, "read": 6.122}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.668, "count": 184, "max_ms": 26.35, "p95_ms": 24.684}, "gstreamer_push": {"avg_ms": 1.119, "count": 184, "max_ms": 1.645, "p95_ms": 1.19}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.867, "count": 184, "max_ms": 38.497, "p95_ms": 33.827}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.027, "count": 184, "max_ms": 84.187, "p95_ms": 80.766}, "yolo11_640_fire_11class": {"avg_ms": 48.506, "count": 184, "max_ms": 67.012, "p95_ms": 62.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.275, "count": 184, "max_ms": 80.73, "p95_ms": 76.571}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.304, "count": 184, "max_ms": 149.844, "p95_ms": 143.856}, "read_interval": {"avg_ms": 138.691, "count": 183, "max_ms": 162.052, "p95_ms": 157.284}, "triton_wait": {"avg_ms": 62.206, "count": 184, "max_ms": 84.199, "p95_ms": 80.778}, "visualize": {"avg_ms": 4.614, "count": 184, "max_ms": 5.746, "p95_ms": 5.006}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12302}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10517.164, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d740ff99c0] error while decoding MB 48 39, bytestream -33 [in#0/rtsp @ 0x55d740214e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d740214e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 195, "h264_decode_error_count": 200, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:31:13+09:00", "last_error_summary": "[h264 @ 0x55d740ff99c0] error while decoding MB 48 39, bytestream -33 [in#0/rtsp @ 0x55d740214e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d740214e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:04:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 1074.763, "last_started_at": "2026-07-01T05:13:18+09:00", "pid": 80320, "process": {"exists": false, "pid": 80320}, "restart_count": 194, "restart_count_per_hour": 8, "rtsp_timeout_count": 318, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 195}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.435, "sample_count": 2741, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1476341, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:31:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.084, "last_frame_pushed_at": "2026-07-01T05:31:13+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T05:31:13+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T05:31:13.433390+09:00", "mtime_epoch": 1782851473.4333901, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222161}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:31:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1758.117, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.747, "pushed": 0.747, "read": 0.804}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.708, "count": 13, "max_ms": 24.985, "p95_ms": 24.985}, "gstreamer_push": {"avg_ms": 1.222, "count": 13, "max_ms": 1.516, "p95_ms": 1.516}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.124, "count": 13, "max_ms": 48.703, "p95_ms": 48.703}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.348, "count": 13, "max_ms": 70.245, "p95_ms": 70.245}, "yolo11_640_fire_11class": {"avg_ms": 44.838, "count": 13, "max_ms": 50.658, "p95_ms": 50.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.169, "count": 13, "max_ms": 64.214, "p95_ms": 64.214}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.016, "count": 13, "max_ms": 96.148, "p95_ms": 96.148}, "read_interval": {"avg_ms": 2471.825, "count": 14, "max_ms": 17271.832, "p95_ms": 17271.832}, "triton_wait": {"avg_ms": 53.361, "count": 13, "max_ms": 70.257, "p95_ms": 70.257}, "visualize": {"avg_ms": 3.366, "count": 13, "max_ms": 3.522, "p95_ms": 3.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.404, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12302}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10516.824, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:31:13+09:00", "last_error_summary": "[h264 @ 0x55d740ff99c0] error while decoding MB 48 39, bytestream -33 [in#0/rtsp @ 0x55d740214e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d740214e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 195, "h264_decode_error_count": 200, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:31:13+09:00", "last_error_summary": "[h264 @ 0x55d740ff99c0] error while decoding MB 48 39, bytestream -33 [in#0/rtsp @ 0x55d740214e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d740214e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:31:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 1074.786, "last_started_at": "2026-07-01T05:13:18+09:00", "pid": 80320, "process": {"exists": false, "pid": 80320}, "restart_count": 194, "restart_count_per_hour": 8, "rtsp_timeout_count": 318, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 195}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.435, "sample_count": 2742, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1476342, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:31:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T05:31:13+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-01T05:31:13+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-01T05:31:13.433390+09:00", "mtime_epoch": 1782851473.4333901, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222161}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:31:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1758.117, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.803, "pushed": 0.803, "read": 0.803}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.025, "count": 14, "max_ms": 24.985, "p95_ms": 24.985}, "gstreamer_push": {"avg_ms": 1.225, "count": 14, "max_ms": 1.516, "p95_ms": 1.516}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.106, "count": 14, "max_ms": 48.703, "p95_ms": 48.703}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.198, "count": 14, "max_ms": 70.245, "p95_ms": 70.245}, "yolo11_640_fire_11class": {"avg_ms": 44.935, "count": 14, "max_ms": 50.658, "p95_ms": 50.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.157, "count": 14, "max_ms": 64.214, "p95_ms": 64.214}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.386, "count": 14, "max_ms": 96.148, "p95_ms": 96.148}, "read_interval": {"avg_ms": 2471.825, "count": 14, "max_ms": 17271.832, "p95_ms": 17271.832}, "triton_wait": {"avg_ms": 53.211, "count": 14, "max_ms": 70.257, "p95_ms": 70.257}, "visualize": {"avg_ms": 3.556, "count": 14, "max_ms": 6.019, "p95_ms": 6.019}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.427, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12302}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10516.941, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 195, "h264_decode_error_count": 200, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:31:13+09:00", "last_error_summary": "[h264 @ 0x55d740ff99c0] error while decoding MB 48 39, bytestream -33 [in#0/rtsp @ 0x55d740214e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d740214e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:31:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 1074.794, "last_started_at": "2026-07-01T05:13:18+09:00", "pid": 80320, "process": {"exists": false, "pid": 80320}, "restart_count": 194, "restart_count_per_hour": 8, "rtsp_timeout_count": 318, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 195}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.435, "sample_count": 2742, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1476342, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:31:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T05:31:13+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T05:31:13+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-01T05:31:13.433390+09:00", "mtime_epoch": 1782851473.4333901, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222161}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:31:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1758.117, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.803, "pushed": 0.803, "read": 0.803}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.025, "count": 14, "max_ms": 24.985, "p95_ms": 24.985}, "gstreamer_push": {"avg_ms": 1.225, "count": 14, "max_ms": 1.516, "p95_ms": 1.516}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.106, "count": 14, "max_ms": 48.703, "p95_ms": 48.703}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.198, "count": 14, "max_ms": 70.245, "p95_ms": 70.245}, "yolo11_640_fire_11class": {"avg_ms": 44.935, "count": 14, "max_ms": 50.658, "p95_ms": 50.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.157, "count": 14, "max_ms": 64.214, "p95_ms": 64.214}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.386, "count": 14, "max_ms": 96.148, "p95_ms": 96.148}, "read_interval": {"avg_ms": 2471.825, "count": 14, "max_ms": 17271.832, "p95_ms": 17271.832}, "triton_wait": {"avg_ms": 53.211, "count": 14, "max_ms": 70.257, "p95_ms": 70.257}, "visualize": {"avg_ms": 3.556, "count": 14, "max_ms": 6.019, "p95_ms": 6.019}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.434, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12302}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10640.605, "limit_mb": 8796093022207.996}, "detail": {"pid": 81005, "restart_count": 195, "start_count": 196}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 195, "h264_decode_error_count": 200, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:31:13+09:00", "last_error_summary": "[h264 @ 0x55d740ff99c0] error while decoding MB 48 39, bytestream -33 [in#0/rtsp @ 0x55d740214e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d740214e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:31:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:31:16+09:00", "pid": 81005, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.938, "name": "ffmpeg", "pid": 81005, "rss_mb": 156.938, "state": "R (running)", "threads": 49}, "restart_count": 195, "restart_count_per_hour": 9, "rtsp_timeout_count": 318, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 196}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 103.44, "sample_count": 2694, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1476342, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:31:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.682, "last_frame_pushed_at": "2026-07-01T05:31:13+09:00", "last_frame_read_age_sec": 4.77, "last_frame_read_at": "2026-07-01T05:31:13+09:00", "mjpeg": {"age_sec": 4.668, "exists": true, "mtime": "2026-07-01T05:31:13.529392+09:00", "mtime_epoch": 1782851473.5293922, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 219848}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:31:18+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1758.117, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.633, "pushed": 0.633, "read": 0.633}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.025, "count": 14, "max_ms": 24.985, "p95_ms": 24.985}, "gstreamer_push": {"avg_ms": 1.225, "count": 14, "max_ms": 1.516, "p95_ms": 1.516}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.106, "count": 14, "max_ms": 48.703, "p95_ms": 48.703}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.198, "count": 14, "max_ms": 70.245, "p95_ms": 70.245}, "yolo11_640_fire_11class": {"avg_ms": 44.935, "count": 14, "max_ms": 50.658, "p95_ms": 50.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.157, "count": 14, "max_ms": 64.214, "p95_ms": 64.214}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.386, "count": 14, "max_ms": 96.148, "p95_ms": 96.148}, "read_interval": {"avg_ms": 2471.825, "count": 14, "max_ms": 17271.832, "p95_ms": 17271.832}, "triton_wait": {"avg_ms": 53.211, "count": 14, "max_ms": 70.257, "p95_ms": 70.257}, "visualize": {"avg_ms": 3.556, "count": 14, "max_ms": 6.019, "p95_ms": 6.019}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.104, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11156}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10585.652, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d040dc5d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d040dc5d80] error while decoding MB 0 42, bytestream 2991 [h264 @ 0x55d040edee00] left block unavailable for requested intra mode [h264 @ 0x55d040edee00] error while decoding MB 0 13, bytestream 244 [h264 @ 0x55d041a96180] error while decoding MB 12 31, bytestream -22 [in#0/rtsp @ 0x55d040daae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d040daae00] Error retrieving a packet from demuxer:", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 389, "h264_decode_error_count": 1271, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:33:48+09:00", "last_error_summary": "[h264 @ 0x55d040dc5d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d040dc5d80] error while decoding MB 0 42, bytestream 2991 [h264 @ 0x55d040edee00] left block unavailable for requested intra mode [h264 @ 0x55d040edee00] error while decoding MB 0 13, bytestream 244 [h264 @ 0x55d041a96180] error while decoding MB 12 31, bytestream -22 [in#0/rtsp @ 0x55d040daae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d040daae00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-01T05:29:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.746, "last_started_at": "2026-07-01T05:29:47+09:00", "pid": 80898, "process": {"exists": false, "pid": 80898}, "restart_count": 423, "restart_count_per_hour": 12, "rtsp_timeout_count": 382, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 424}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.95, "sample_count": 2020, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1338788, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:33:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T05:33:48+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T05:33:48+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T05:33:48.164836+09:00", "mtime_epoch": 1782851628.164836, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159958}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:33:48+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5003.672, "name": "celery", "pid": 267, "rss_mb": 5003.672, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 81, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.705, "pushed": 2.705, "read": 2.705}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.538, "count": 81, "max_ms": 31.761, "p95_ms": 29.271}, "gstreamer_push": {"avg_ms": 1.108, "count": 81, "max_ms": 1.292, "p95_ms": 1.162}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.544, "count": 81, "max_ms": 38.9, "p95_ms": 35.875}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.207, "count": 81, "max_ms": 81.443, "p95_ms": 79.051}, "yolo11_640_fire_11class": {"avg_ms": 47.372, "count": 81, "max_ms": 64.674, "p95_ms": 61.643}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.947, "count": 81, "max_ms": 80.26, "p95_ms": 77.059}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.852, "count": 81, "max_ms": 145.98, "p95_ms": 143.885}, "read_interval": {"avg_ms": 370.743, "count": 81, "max_ms": 10123.226, "p95_ms": 157.26}, "triton_wait": {"avg_ms": 61.437, "count": 81, "max_ms": 81.454, "p95_ms": 79.445}, "visualize": {"avg_ms": 4.602, "count": 81, "max_ms": 5.766, "p95_ms": 5.134}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11156}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10585.117, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:33:48+09:00", "last_error_summary": "[h264 @ 0x55d040dc5d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d040dc5d80] error while decoding MB 0 42, bytestream 2991 [h264 @ 0x55d040edee00] left block unavailable for requested intra mode [h264 @ 0x55d040edee00] error while decoding MB 0 13, bytestream 244 [h264 @ 0x55d041a96180] error while decoding MB 12 31, bytestream -22 [in#0/rtsp @ 0x55d040daae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d040daae00] Error retrieving a packet from demuxer:", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 389, "h264_decode_error_count": 1271, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:33:48+09:00", "last_error_summary": "[h264 @ 0x55d040dc5d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d040dc5d80] error while decoding MB 0 42, bytestream 2991 [h264 @ 0x55d040edee00] left block unavailable for requested intra mode [h264 @ 0x55d040edee00] error while decoding MB 0 13, bytestream 244 [h264 @ 0x55d041a96180] error while decoding MB 12 31, bytestream -22 [in#0/rtsp @ 0x55d040daae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d040daae00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-01T05:33:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.847, "last_started_at": "2026-07-01T05:29:47+09:00", "pid": 80898, "process": {"exists": false, "pid": 80898}, "restart_count": 423, "restart_count_per_hour": 12, "rtsp_timeout_count": 382, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 424}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.957, "sample_count": 2020, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1338789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:33:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T05:33:48+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-01T05:33:48+09:00", "mjpeg": {"age_sec": 0.146, "exists": true, "mtime": "2026-07-01T05:33:48.164836+09:00", "mtime_epoch": 1782851628.164836, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159958}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:33:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5003.672, "name": "celery", "pid": 267, "rss_mb": 5003.672, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.709, "pushed": 2.709, "read": 2.675}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.553, "count": 81, "max_ms": 31.761, "p95_ms": 29.271}, "gstreamer_push": {"avg_ms": 1.107, "count": 81, "max_ms": 1.292, "p95_ms": 1.162}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.521, "count": 81, "max_ms": 38.9, "p95_ms": 35.875}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.287, "count": 81, "max_ms": 82.966, "p95_ms": 80.268}, "yolo11_640_fire_11class": {"avg_ms": 47.478, "count": 81, "max_ms": 65.036, "p95_ms": 62.576}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.044, "count": 81, "max_ms": 80.26, "p95_ms": 77.635}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.928, "count": 81, "max_ms": 146.063, "p95_ms": 143.95}, "read_interval": {"avg_ms": 373.458, "count": 80, "max_ms": 10123.226, "p95_ms": 157.26}, "triton_wait": {"avg_ms": 61.517, "count": 81, "max_ms": 82.977, "p95_ms": 80.281}, "visualize": {"avg_ms": 4.6, "count": 81, "max_ms": 5.766, "p95_ms": 5.134}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11156}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10585.105, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 389, "h264_decode_error_count": 1271, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:33:48+09:00", "last_error_summary": "[h264 @ 0x55d040dc5d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d040dc5d80] error while decoding MB 0 42, bytestream 2991 [h264 @ 0x55d040edee00] left block unavailable for requested intra mode [h264 @ 0x55d040edee00] error while decoding MB 0 13, bytestream 244 [h264 @ 0x55d041a96180] error while decoding MB 12 31, bytestream -22 [in#0/rtsp @ 0x55d040daae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d040daae00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-01T05:33:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.851, "last_started_at": "2026-07-01T05:29:47+09:00", "pid": 80898, "process": {"exists": false, "pid": 80898}, "restart_count": 423, "restart_count_per_hour": 12, "rtsp_timeout_count": 382, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 424}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.957, "sample_count": 2020, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1338789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:33:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T05:33:48+09:00", "last_frame_read_age_sec": 0.157, "last_frame_read_at": "2026-07-01T05:33:48+09:00", "mjpeg": {"age_sec": 0.151, "exists": true, "mtime": "2026-07-01T05:33:48.164836+09:00", "mtime_epoch": 1782851628.164836, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159958}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:33:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5003.672, "name": "celery", "pid": 267, "rss_mb": 5003.672, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.708, "pushed": 2.708, "read": 2.675}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.553, "count": 81, "max_ms": 31.761, "p95_ms": 29.271}, "gstreamer_push": {"avg_ms": 1.107, "count": 81, "max_ms": 1.292, "p95_ms": 1.162}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.521, "count": 81, "max_ms": 38.9, "p95_ms": 35.875}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.287, "count": 81, "max_ms": 82.966, "p95_ms": 80.268}, "yolo11_640_fire_11class": {"avg_ms": 47.478, "count": 81, "max_ms": 65.036, "p95_ms": 62.576}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.044, "count": 81, "max_ms": 80.26, "p95_ms": 77.635}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.928, "count": 81, "max_ms": 146.063, "p95_ms": 143.95}, "read_interval": {"avg_ms": 373.458, "count": 80, "max_ms": 10123.226, "p95_ms": 157.26}, "triton_wait": {"avg_ms": 61.517, "count": 81, "max_ms": 82.977, "p95_ms": 80.281}, "visualize": {"avg_ms": 4.6, "count": 81, "max_ms": 5.766, "p95_ms": 5.134}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11156}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10693.176, "limit_mb": 8796093022207.996}, "detail": {"pid": 81157, "restart_count": 424, "start_count": 425}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 389, "h264_decode_error_count": 1271, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:33:48+09:00", "last_error_summary": "[h264 @ 0x55d040dc5d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d040dc5d80] error while decoding MB 0 42, bytestream 2991 [h264 @ 0x55d040edee00] left block unavailable for requested intra mode [h264 @ 0x55d040edee00] error while decoding MB 0 13, bytestream 244 [h264 @ 0x55d041a96180] error while decoding MB 12 31, bytestream -22 [in#0/rtsp @ 0x55d040daae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d040daae00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-01T05:33:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:33:50+09:00", "pid": 81157, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.727, "name": "ffmpeg", "pid": 81157, "rss_mb": 138.727, "state": "S (sleeping)", "threads": 106}, "restart_count": 424, "restart_count_per_hour": 13, "rtsp_timeout_count": 382, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 425}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.006, "sample_count": 1987, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1338789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:33:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.471, "last_frame_pushed_at": "2026-07-01T05:33:48+09:00", "last_frame_read_age_sec": 4.619, "last_frame_read_at": "2026-07-01T05:33:48+09:00", "mjpeg": {"age_sec": 4.457, "exists": true, "mtime": "2026-07-01T05:33:48.320839+09:00", "mtime_epoch": 1782851628.3208392, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160033}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:33:52+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5003.672, "name": "celery", "pid": 267, "rss_mb": 5003.672, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.64, "pushed": 1.64, "read": 1.607}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.411, "count": 49, "max_ms": 31.761, "p95_ms": 27.62}, "gstreamer_push": {"avg_ms": 1.111, "count": 49, "max_ms": 1.263, "p95_ms": 1.17}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.724, "count": 49, "max_ms": 37.291, "p95_ms": 35.101}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.136, "count": 49, "max_ms": 82.966, "p95_ms": 79.051}, "yolo11_640_fire_11class": {"avg_ms": 47.587, "count": 49, "max_ms": 65.036, "p95_ms": 62.576}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.795, "count": 49, "max_ms": 79.974, "p95_ms": 79.434}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.283, "count": 49, "max_ms": 146.063, "p95_ms": 143.885}, "read_interval": {"avg_ms": 528.869, "count": 48, "max_ms": 10123.226, "p95_ms": 156.162}, "triton_wait": {"avg_ms": 61.225, "count": 49, "max_ms": 82.977, "p95_ms": 79.445}, "visualize": {"avg_ms": 4.614, "count": 49, "max_ms": 5.674, "p95_ms": 5.134}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11169}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10588.211, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x564f31d65300] left block unavailable for requested intra mode [h264 @ 0x564f31d65300] error while decoding MB 0 20, bytestream 518 [in#0/rtsp @ 0x564f31ce1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564f31ce1e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 390, "h264_decode_error_count": 1272, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:37:46+09:00", "last_error_summary": "[h264 @ 0x564f31d65300] left block unavailable for requested intra mode [h264 @ 0x564f31d65300] error while decoding MB 0 20, bytestream 518 [in#0/rtsp @ 0x564f31ce1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564f31ce1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:33:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 235.976, "last_started_at": "2026-07-01T05:33:50+09:00", "pid": 81157, "process": {"exists": false, "pid": 81157}, "restart_count": 424, "restart_count_per_hour": 12, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 425}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.18, "sample_count": 1850, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1340330, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:37:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T05:37:46+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T05:37:46+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T05:37:46.710042+09:00", "mtime_epoch": 1782851866.7100415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170124}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:37:46+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5004.703, "name": "celery", "pid": 267, "rss_mb": 5004.703, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.768, "pushed": 2.768, "read": 2.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.686, "count": 82, "max_ms": 28.189, "p95_ms": 25.596}, "gstreamer_push": {"avg_ms": 1.103, "count": 83, "max_ms": 1.242, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.933, "count": 82, "max_ms": 41.471, "p95_ms": 34.814}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.371, "count": 82, "max_ms": 81.449, "p95_ms": 79.967}, "yolo11_640_fire_11class": {"avg_ms": 47.646, "count": 82, "max_ms": 63.354, "p95_ms": 58.934}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.442, "count": 82, "max_ms": 78.753, "p95_ms": 74.836}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.601, "count": 83, "max_ms": 146.491, "p95_ms": 143.324}, "read_interval": {"avg_ms": 362.138, "count": 83, "max_ms": 18945.307, "p95_ms": 155.388}, "triton_wait": {"avg_ms": 59.525, "count": 82, "max_ms": 81.46, "p95_ms": 79.978}, "visualize": {"avg_ms": 4.633, "count": 82, "max_ms": 6.633, "p95_ms": 4.898}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11169}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10587.637, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:37:46+09:00", "last_error_summary": "[h264 @ 0x564f31d65300] left block unavailable for requested intra mode [h264 @ 0x564f31d65300] error while decoding MB 0 20, bytestream 518 [in#0/rtsp @ 0x564f31ce1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564f31ce1e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 390, "h264_decode_error_count": 1272, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:37:46+09:00", "last_error_summary": "[h264 @ 0x564f31d65300] left block unavailable for requested intra mode [h264 @ 0x564f31d65300] error while decoding MB 0 20, bytestream 518 [in#0/rtsp @ 0x564f31ce1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564f31ce1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:37:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 236.065, "last_started_at": "2026-07-01T05:33:50+09:00", "pid": 81157, "process": {"exists": false, "pid": 81157}, "restart_count": 424, "restart_count_per_hour": 12, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 425}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.18, "sample_count": 1851, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1340331, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:37:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T05:37:46+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T05:37:46+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-01T05:37:46.710042+09:00", "mtime_epoch": 1782851866.7100415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170124}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:37:46+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5004.703, "name": "celery", "pid": 267, "rss_mb": 5004.703, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 82, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.772, "pushed": 2.772, "read": 2.738}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.67, "count": 83, "max_ms": 28.189, "p95_ms": 25.596}, "gstreamer_push": {"avg_ms": 1.104, "count": 83, "max_ms": 1.242, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.969, "count": 83, "max_ms": 41.471, "p95_ms": 34.912}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.538, "count": 83, "max_ms": 81.449, "p95_ms": 79.967}, "yolo11_640_fire_11class": {"avg_ms": 47.747, "count": 83, "max_ms": 63.354, "p95_ms": 58.934}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.614, "count": 83, "max_ms": 78.753, "p95_ms": 74.836}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.709, "count": 83, "max_ms": 146.491, "p95_ms": 143.324}, "read_interval": {"avg_ms": 364.864, "count": 82, "max_ms": 18945.307, "p95_ms": 155.388}, "triton_wait": {"avg_ms": 59.69, "count": 83, "max_ms": 81.46, "p95_ms": 79.978}, "visualize": {"avg_ms": 4.632, "count": 83, "max_ms": 6.633, "p95_ms": 4.898}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11169}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10587.629, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 390, "h264_decode_error_count": 1272, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:37:46+09:00", "last_error_summary": "[h264 @ 0x564f31d65300] left block unavailable for requested intra mode [h264 @ 0x564f31d65300] error while decoding MB 0 20, bytestream 518 [in#0/rtsp @ 0x564f31ce1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564f31ce1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:37:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 236.07, "last_started_at": "2026-07-01T05:33:50+09:00", "pid": 81157, "process": {"exists": false, "pid": 81157}, "restart_count": 424, "restart_count_per_hour": 12, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 425}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.18, "sample_count": 1851, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1340331, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:37:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T05:37:46+09:00", "last_frame_read_age_sec": 0.147, "last_frame_read_at": "2026-07-01T05:37:46+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-01T05:37:46.710042+09:00", "mtime_epoch": 1782851866.7100415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170124}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:37:46+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5004.703, "name": "celery", "pid": 267, "rss_mb": 5004.703, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 82, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.771, "pushed": 2.771, "read": 2.738}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.67, "count": 83, "max_ms": 28.189, "p95_ms": 25.596}, "gstreamer_push": {"avg_ms": 1.104, "count": 83, "max_ms": 1.242, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.969, "count": 83, "max_ms": 41.471, "p95_ms": 34.912}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.538, "count": 83, "max_ms": 81.449, "p95_ms": 79.967}, "yolo11_640_fire_11class": {"avg_ms": 47.747, "count": 83, "max_ms": 63.354, "p95_ms": 58.934}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.614, "count": 83, "max_ms": 78.753, "p95_ms": 74.836}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.709, "count": 83, "max_ms": 146.491, "p95_ms": 143.324}, "read_interval": {"avg_ms": 364.864, "count": 82, "max_ms": 18945.307, "p95_ms": 155.388}, "triton_wait": {"avg_ms": 59.69, "count": 83, "max_ms": 81.46, "p95_ms": 79.978}, "visualize": {"avg_ms": 4.632, "count": 83, "max_ms": 6.633, "p95_ms": 4.898}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11169}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10696.645, "limit_mb": 8796093022207.996}, "detail": {"pid": 81264, "restart_count": 425, "start_count": 426}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 390, "h264_decode_error_count": 1272, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:37:46+09:00", "last_error_summary": "[h264 @ 0x564f31d65300] left block unavailable for requested intra mode [h264 @ 0x564f31d65300] error while decoding MB 0 20, bytestream 518 [in#0/rtsp @ 0x564f31ce1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564f31ce1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:37:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:37:49+09:00", "pid": 81264, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.078, "name": "ffmpeg", "pid": 81264, "rss_mb": 139.078, "state": "S (sleeping)", "threads": 106}, "restart_count": 425, "restart_count_per_hour": 13, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 426}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.24, "sample_count": 1814, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1340331, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:37:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 5.022, "last_frame_pushed_at": "2026-07-01T05:37:46+09:00", "last_frame_read_age_sec": 5.159, "last_frame_read_at": "2026-07-01T05:37:46+09:00", "mjpeg": {"age_sec": 5.006, "exists": true, "mtime": "2026-07-01T05:37:46.854045+09:00", "mtime_epoch": 1782851866.8540447, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167369}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:37:51+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5004.703, "name": "celery", "pid": 267, "rss_mb": 5004.703, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 46, "pushed_frames": 46, "read_frames": 45, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.538, "pushed": 1.538, "read": 1.504}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.736, "count": 46, "max_ms": 28.189, "p95_ms": 25.596}, "gstreamer_push": {"avg_ms": 1.099, "count": 46, "max_ms": 1.194, "p95_ms": 1.175}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.832, "count": 46, "max_ms": 40.424, "p95_ms": 34.912}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.91, "count": 46, "max_ms": 80.966, "p95_ms": 78.388}, "yolo11_640_fire_11class": {"avg_ms": 47.625, "count": 46, "max_ms": 63.354, "p95_ms": 58.934}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.301, "count": 46, "max_ms": 78.614, "p95_ms": 72.096}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.201, "count": 46, "max_ms": 146.491, "p95_ms": 140.675}, "read_interval": {"avg_ms": 552.643, "count": 45, "max_ms": 18945.307, "p95_ms": 155.854}, "triton_wait": {"avg_ms": 59.174, "count": 46, "max_ms": 80.978, "p95_ms": 78.4}, "visualize": {"avg_ms": 4.61, "count": 46, "max_ms": 6.144, "p95_ms": 4.841}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11181}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10589.773, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fef9abac80] error while decoding MB 73 33, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 391, "h264_decode_error_count": 1273, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:41:03+09:00", "last_error_summary": "[h264 @ 0x55fef9abac80] error while decoding MB 73 33, bytestream -5", "last_exit_at": "2026-07-01T05:37:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.664, "last_started_at": "2026-07-01T05:37:49+09:00", "pid": 81264, "process": {"exists": false, "pid": 81264}, "restart_count": 425, "restart_count_per_hour": 13, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 426}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.434, "sample_count": 2026, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1341758, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:41:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T05:41:03+09:00", "last_frame_read_age_sec": 0.046, "last_frame_read_at": "2026-07-01T05:41:03+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T05:41:03.482269+09:00", "mtime_epoch": 1782852063.482269, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:41:03+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5005.219, "name": "celery", "pid": 267, "rss_mb": 5005.219, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 256, "pushed_frames": 256, "read_frames": 256, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.539, "pushed": 8.539, "read": 8.539}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.142, "count": 255, "max_ms": 30.01, "p95_ms": 24.727}, "gstreamer_push": {"avg_ms": 1.191, "count": 256, "max_ms": 1.444, "p95_ms": 1.301}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.193, "count": 255, "max_ms": 42.37, "p95_ms": 35.213}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.039, "count": 255, "max_ms": 86.341, "p95_ms": 80.842}, "yolo11_640_fire_11class": {"avg_ms": 48.39, "count": 255, "max_ms": 67.041, "p95_ms": 63.27}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.385, "count": 255, "max_ms": 79.361, "p95_ms": 76.814}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.65, "count": 256, "max_ms": 152.622, "p95_ms": 137.391}, "read_interval": {"avg_ms": 117.396, "count": 256, "max_ms": 164.14, "p95_ms": 151.461}, "triton_wait": {"avg_ms": 65.051, "count": 255, "max_ms": 86.353, "p95_ms": 80.854}, "visualize": {"avg_ms": 3.769, "count": 255, "max_ms": 5.166, "p95_ms": 4.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11181}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10589.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:41:03+09:00", "last_error_summary": "[h264 @ 0x55fef9abac80] error while decoding MB 73 33, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 391, "h264_decode_error_count": 1273, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:41:03+09:00", "last_error_summary": "[h264 @ 0x55fef9abac80] error while decoding MB 73 33, bytestream -5", "last_exit_at": "2026-07-01T05:41:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.713, "last_started_at": "2026-07-01T05:37:49+09:00", "pid": 81264, "process": {"exists": false, "pid": 81264}, "restart_count": 425, "restart_count_per_hour": 13, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 426}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.434, "sample_count": 2027, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1341759, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:41:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T05:41:03+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-01T05:41:03+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T05:41:03.482269+09:00", "mtime_epoch": 1782852063.482269, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:41:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5005.219, "name": "celery", "pid": 267, "rss_mb": 5005.219, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 256, "pushed_frames": 256, "read_frames": 255, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.565, "pushed": 8.565, "read": 8.532}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.14, "count": 256, "max_ms": 30.01, "p95_ms": 24.727}, "gstreamer_push": {"avg_ms": 1.192, "count": 256, "max_ms": 1.444, "p95_ms": 1.301}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.192, "count": 256, "max_ms": 42.37, "p95_ms": 35.213}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.013, "count": 256, "max_ms": 86.341, "p95_ms": 80.842}, "yolo11_640_fire_11class": {"avg_ms": 48.362, "count": 256, "max_ms": 67.041, "p95_ms": 63.27}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.368, "count": 256, "max_ms": 79.361, "p95_ms": 76.814}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.517, "count": 256, "max_ms": 152.622, "p95_ms": 137.391}, "read_interval": {"avg_ms": 117.336, "count": 255, "max_ms": 164.14, "p95_ms": 151.461}, "triton_wait": {"avg_ms": 65.025, "count": 256, "max_ms": 86.353, "p95_ms": 80.854}, "visualize": {"avg_ms": 3.767, "count": 256, "max_ms": 5.166, "p95_ms": 4.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11181}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10589.383, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 391, "h264_decode_error_count": 1273, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:41:03+09:00", "last_error_summary": "[h264 @ 0x55fef9abac80] error while decoding MB 73 33, bytestream -5", "last_exit_at": "2026-07-01T05:41:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 193.721, "last_started_at": "2026-07-01T05:37:49+09:00", "pid": 81264, "process": {"exists": false, "pid": 81264}, "restart_count": 425, "restart_count_per_hour": 13, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 426}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.434, "sample_count": 2027, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1341759, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:41:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T05:41:03+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T05:41:03+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-01T05:41:03.482269+09:00", "mtime_epoch": 1782852063.482269, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:41:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5005.219, "name": "celery", "pid": 267, "rss_mb": 5005.219, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 256, "pushed_frames": 256, "read_frames": 255, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.563, "pushed": 8.563, "read": 8.53}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.14, "count": 256, "max_ms": 30.01, "p95_ms": 24.727}, "gstreamer_push": {"avg_ms": 1.192, "count": 256, "max_ms": 1.444, "p95_ms": 1.301}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.192, "count": 256, "max_ms": 42.37, "p95_ms": 35.213}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.013, "count": 256, "max_ms": 86.341, "p95_ms": 80.842}, "yolo11_640_fire_11class": {"avg_ms": 48.362, "count": 256, "max_ms": 67.041, "p95_ms": 63.27}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.368, "count": 256, "max_ms": 79.361, "p95_ms": 76.814}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.517, "count": 256, "max_ms": 152.622, "p95_ms": 137.391}, "read_interval": {"avg_ms": 117.336, "count": 255, "max_ms": 164.14, "p95_ms": 151.461}, "triton_wait": {"avg_ms": 65.025, "count": 256, "max_ms": 86.353, "p95_ms": 80.854}, "visualize": {"avg_ms": 3.767, "count": 256, "max_ms": 5.166, "p95_ms": 4.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11181}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10689.305, "limit_mb": 8796093022207.996}, "detail": {"pid": 81371, "restart_count": 426, "start_count": 427}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 391, "h264_decode_error_count": 1273, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:41:03+09:00", "last_error_summary": "[h264 @ 0x55fef9abac80] error while decoding MB 73 33, bytestream -5", "last_exit_at": "2026-07-01T05:41:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:41:06+09:00", "pid": 81371, "process": {"exists": true, "fd_count": 5, "hwm_mb": 130.527, "name": "ffmpeg", "pid": 81371, "rss_mb": 130.527, "state": "S (sleeping)", "threads": 106}, "restart_count": 426, "restart_count_per_hour": 14, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 427}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.446, "sample_count": 1990, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1341759, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:41:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 5.001, "last_frame_pushed_at": "2026-07-01T05:41:03+09:00", "last_frame_read_age_sec": 5.09, "last_frame_read_at": "2026-07-01T05:41:03+09:00", "mjpeg": {"age_sec": 4.985, "exists": true, "mtime": "2026-07-01T05:41:03.582271+09:00", "mtime_epoch": 1782852063.582271, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165083}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:41:08+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5005.219, "name": "celery", "pid": 267, "rss_mb": 5005.219, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 221, "pushed_frames": 221, "read_frames": 220, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.381, "pushed": 7.381, "read": 7.348}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.876, "count": 221, "max_ms": 30.01, "p95_ms": 24.054}, "gstreamer_push": {"avg_ms": 1.202, "count": 221, "max_ms": 1.444, "p95_ms": 1.302}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.391, "count": 221, "max_ms": 42.37, "p95_ms": 35.731}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.897, "count": 221, "max_ms": 85.698, "p95_ms": 79.868}, "yolo11_640_fire_11class": {"avg_ms": 48.19, "count": 221, "max_ms": 67.041, "p95_ms": 62.789}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.138, "count": 221, "max_ms": 79.361, "p95_ms": 76.574}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.509, "count": 221, "max_ms": 148.062, "p95_ms": 130.268}, "read_interval": {"avg_ms": 113.548, "count": 220, "max_ms": 160.139, "p95_ms": 142.35}, "triton_wait": {"avg_ms": 64.91, "count": 221, "max_ms": 85.71, "p95_ms": 79.879}, "visualize": {"avg_ms": 3.634, "count": 221, "max_ms": 5.108, "p95_ms": 4.724}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10528.0, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e63ce83b40] error while decoding MB 63 46, bytestream -39 [in#0/rtsp @ 0x55e63cd14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e63cd14e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 196, "h264_decode_error_count": 201, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:43:49+09:00", "last_error_summary": "[h264 @ 0x55e63ce83b40] error while decoding MB 63 46, bytestream -39 [in#0/rtsp @ 0x55e63cd14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e63cd14e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:31:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 752.929, "last_started_at": "2026-07-01T05:31:16+09:00", "pid": 81005, "process": {"exists": false, "pid": 81005}, "restart_count": 195, "restart_count_per_hour": 4, "rtsp_timeout_count": 320, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 196}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.107, "sample_count": 2466, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1483321, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:43:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-01T05:43:49+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-01T05:43:49+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-01T05:43:49.069795+09:00", "mtime_epoch": 1782852229.0697954, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 203538}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:43:49+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1761.469, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 15, "pushed_frames": 15, "read_frames": 15, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.501, "pushed": 0.501, "read": 0.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.605, "count": 15, "max_ms": 19.741, "p95_ms": 19.741}, "gstreamer_push": {"avg_ms": 1.242, "count": 15, "max_ms": 1.33, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.427, "count": 15, "max_ms": 56.865, "p95_ms": 56.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.799, "count": 15, "max_ms": 82.123, "p95_ms": 82.123}, "yolo11_640_fire_11class": {"avg_ms": 57.969, "count": 15, "max_ms": 64.937, "p95_ms": 64.937}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.712, "count": 15, "max_ms": 78.656, "p95_ms": 78.656}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.687, "count": 15, "max_ms": 108.458, "p95_ms": 108.458}, "read_interval": {"avg_ms": 1999.556, "count": 15, "max_ms": 16336.255, "p95_ms": 16336.255}, "triton_wait": {"avg_ms": 70.812, "count": 15, "max_ms": 82.135, "p95_ms": 82.135}, "visualize": {"avg_ms": 3.444, "count": 15, "max_ms": 3.781, "p95_ms": 3.781}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10527.457, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:43:49+09:00", "last_error_summary": "[h264 @ 0x55e63ce83b40] error while decoding MB 63 46, bytestream -39 [in#0/rtsp @ 0x55e63cd14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e63cd14e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 196, "h264_decode_error_count": 201, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:43:49+09:00", "last_error_summary": "[h264 @ 0x55e63ce83b40] error while decoding MB 63 46, bytestream -39 [in#0/rtsp @ 0x55e63cd14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e63cd14e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:43:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 752.983, "last_started_at": "2026-07-01T05:31:16+09:00", "pid": 81005, "process": {"exists": false, "pid": 81005}, "restart_count": 195, "restart_count_per_hour": 4, "rtsp_timeout_count": 320, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 196}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.107, "sample_count": 2467, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1483322, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:43:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T05:43:49+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-01T05:43:49+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T05:43:49.069795+09:00", "mtime_epoch": 1782852229.0697954, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 203538}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:43:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1761.469, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 15, "pushed_frames": 16, "read_frames": 15, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.5, "pushed": 0.533, "read": 0.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.579, "count": 15, "max_ms": 19.741, "p95_ms": 19.741}, "gstreamer_push": {"avg_ms": 1.243, "count": 16, "max_ms": 1.33, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.198, "count": 15, "max_ms": 56.865, "p95_ms": 56.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.851, "count": 15, "max_ms": 82.123, "p95_ms": 82.123}, "yolo11_640_fire_11class": {"avg_ms": 57.905, "count": 15, "max_ms": 64.015, "p95_ms": 64.015}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.62, "count": 15, "max_ms": 78.656, "p95_ms": 78.656}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.704, "count": 15, "max_ms": 108.458, "p95_ms": 108.458}, "read_interval": {"avg_ms": 1999.556, "count": 15, "max_ms": 16336.255, "p95_ms": 16336.255}, "triton_wait": {"avg_ms": 70.863, "count": 15, "max_ms": 82.135, "p95_ms": 82.135}, "visualize": {"avg_ms": 3.449, "count": 15, "max_ms": 3.781, "p95_ms": 3.781}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10527.375, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 196, "h264_decode_error_count": 201, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:43:49+09:00", "last_error_summary": "[h264 @ 0x55e63ce83b40] error while decoding MB 63 46, bytestream -39 [in#0/rtsp @ 0x55e63cd14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e63cd14e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:43:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 752.989, "last_started_at": "2026-07-01T05:31:16+09:00", "pid": 81005, "process": {"exists": false, "pid": 81005}, "restart_count": 195, "restart_count_per_hour": 4, "rtsp_timeout_count": 320, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 196}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.107, "sample_count": 2467, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1483322, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:43:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T05:43:49+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-01T05:43:49+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-01T05:43:49.069795+09:00", "mtime_epoch": 1782852229.0697954, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 203538}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:43:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1761.469, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 15, "pushed_frames": 15, "read_frames": 15, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.5, "pushed": 0.5, "read": 0.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.579, "count": 15, "max_ms": 19.741, "p95_ms": 19.741}, "gstreamer_push": {"avg_ms": 1.244, "count": 15, "max_ms": 1.33, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.198, "count": 15, "max_ms": 56.865, "p95_ms": 56.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.851, "count": 15, "max_ms": 82.123, "p95_ms": 82.123}, "yolo11_640_fire_11class": {"avg_ms": 57.905, "count": 15, "max_ms": 64.015, "p95_ms": 64.015}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.62, "count": 15, "max_ms": 78.656, "p95_ms": 78.656}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.704, "count": 15, "max_ms": 108.458, "p95_ms": 108.458}, "read_interval": {"avg_ms": 1999.556, "count": 15, "max_ms": 16336.255, "p95_ms": 16336.255}, "triton_wait": {"avg_ms": 70.863, "count": 15, "max_ms": 82.135, "p95_ms": 82.135}, "visualize": {"avg_ms": 3.449, "count": 15, "max_ms": 3.781, "p95_ms": 3.781}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10548.719, "limit_mb": 8796093022207.996}, "detail": {"pid": 81478, "restart_count": 196, "start_count": 197}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 196, "h264_decode_error_count": 201, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:43:49+09:00", "last_error_summary": "[h264 @ 0x55e63ce83b40] error while decoding MB 63 46, bytestream -39 [in#0/rtsp @ 0x55e63cd14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e63cd14e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:43:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:43:51+09:00", "pid": 81478, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.676, "name": "ffmpeg", "pid": 81478, "rss_mb": 57.676, "state": "S (sleeping)", "threads": 1}, "restart_count": 196, "restart_count_per_hour": 5, "rtsp_timeout_count": 320, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 197}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.116, "sample_count": 2419, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1483322, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:43:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.738, "last_frame_pushed_at": "2026-07-01T05:43:49+09:00", "last_frame_read_age_sec": 4.847, "last_frame_read_at": "2026-07-01T05:43:49+09:00", "mjpeg": {"age_sec": 4.721, "exists": true, "mtime": "2026-07-01T05:43:49.189798+09:00", "mtime_epoch": 1782852229.1897979, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 202493}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:43:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1761.469, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.622, "pushed": 0.622, "read": 0.622}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.616, "count": 14, "max_ms": 19.741, "p95_ms": 19.741}, "gstreamer_push": {"avg_ms": 1.246, "count": 14, "max_ms": 1.33, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.267, "count": 14, "max_ms": 56.865, "p95_ms": 56.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.131, "count": 14, "max_ms": 82.123, "p95_ms": 82.123}, "yolo11_640_fire_11class": {"avg_ms": 57.469, "count": 14, "max_ms": 63.98, "p95_ms": 63.98}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.801, "count": 14, "max_ms": 78.656, "p95_ms": 78.656}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.021, "count": 14, "max_ms": 108.458, "p95_ms": 108.458}, "read_interval": {"avg_ms": 2133.706, "count": 14, "max_ms": 16336.255, "p95_ms": 16336.255}, "triton_wait": {"avg_ms": 70.143, "count": 14, "max_ms": 82.135, "p95_ms": 82.135}, "visualize": {"avg_ms": 3.451, "count": 14, "max_ms": 3.781, "p95_ms": 3.781}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.52, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11203}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10592.902, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558e4e42f980] error while decoding MB 17 34, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 392, "h264_decode_error_count": 1274, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:46:30+09:00", "last_error_summary": "[h264 @ 0x558e4e42f980] error while decoding MB 17 34, bytestream -5", "last_exit_at": "2026-07-01T05:41:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 323.672, "last_started_at": "2026-07-01T05:41:06+09:00", "pid": 81371, "process": {"exists": false, "pid": 81371}, "restart_count": 426, "restart_count_per_hour": 13, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 427}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.694, "sample_count": 2530, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1344469, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:46:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-01T05:46:30+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T05:46:30+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-01T05:46:30.189207+09:00", "mtime_epoch": 1782852390.1892068, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160132}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:46:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5006.508, "name": "celery", "pid": 267, "rss_mb": 5006.508, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 248, "pushed_frames": 248, "read_frames": 248, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.277, "pushed": 8.277, "read": 8.277}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.376, "count": 248, "max_ms": 24.945, "p95_ms": 21.41}, "gstreamer_push": {"avg_ms": 1.218, "count": 248, "max_ms": 1.392, "p95_ms": 1.301}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.117, "count": 248, "max_ms": 45.274, "p95_ms": 42.442}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.864, "count": 248, "max_ms": 85.875, "p95_ms": 80.596}, "yolo11_640_fire_11class": {"avg_ms": 61.152, "count": 248, "max_ms": 66.347, "p95_ms": 64.793}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.746, "count": 248, "max_ms": 81.342, "p95_ms": 79.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.947, "count": 248, "max_ms": 286.938, "p95_ms": 109.242}, "read_interval": {"avg_ms": 121.038, "count": 248, "max_ms": 299.377, "p95_ms": 125.158}, "triton_wait": {"avg_ms": 76.45, "count": 248, "max_ms": 85.887, "p95_ms": 80.874}, "visualize": {"avg_ms": 3.453, "count": 248, "max_ms": 4.671, "p95_ms": 3.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11203}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10592.441, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:46:30+09:00", "last_error_summary": "[h264 @ 0x558e4e42f980] error while decoding MB 17 34, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 392, "h264_decode_error_count": 1274, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:46:30+09:00", "last_error_summary": "[h264 @ 0x558e4e42f980] error while decoding MB 17 34, bytestream -5", "last_exit_at": "2026-07-01T05:46:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 323.734, "last_started_at": "2026-07-01T05:41:06+09:00", "pid": 81371, "process": {"exists": false, "pid": 81371}, "restart_count": 426, "restart_count_per_hour": 13, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 427}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.694, "sample_count": 2531, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1344470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:46:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T05:46:30+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-01T05:46:30+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-01T05:46:30.189207+09:00", "mtime_epoch": 1782852390.1892068, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160132}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:46:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5006.508, "name": "celery", "pid": 267, "rss_mb": 5006.508, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 248, "pushed_frames": 248, "read_frames": 247, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.293, "pushed": 8.293, "read": 8.26}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.373, "count": 248, "max_ms": 24.945, "p95_ms": 21.41}, "gstreamer_push": {"avg_ms": 1.218, "count": 248, "max_ms": 1.392, "p95_ms": 1.301}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.086, "count": 248, "max_ms": 45.274, "p95_ms": 42.442}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.857, "count": 248, "max_ms": 85.875, "p95_ms": 80.596}, "yolo11_640_fire_11class": {"avg_ms": 61.158, "count": 248, "max_ms": 66.347, "p95_ms": 64.908}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.751, "count": 248, "max_ms": 81.342, "p95_ms": 79.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.942, "count": 248, "max_ms": 286.938, "p95_ms": 109.242}, "read_interval": {"avg_ms": 121.034, "count": 247, "max_ms": 299.377, "p95_ms": 125.158}, "triton_wait": {"avg_ms": 76.449, "count": 248, "max_ms": 85.887, "p95_ms": 80.874}, "visualize": {"avg_ms": 3.453, "count": 248, "max_ms": 4.671, "p95_ms": 3.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11203}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10592.402, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 392, "h264_decode_error_count": 1274, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:46:30+09:00", "last_error_summary": "[h264 @ 0x558e4e42f980] error while decoding MB 17 34, bytestream -5", "last_exit_at": "2026-07-01T05:46:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 323.742, "last_started_at": "2026-07-01T05:41:06+09:00", "pid": 81371, "process": {"exists": false, "pid": 81371}, "restart_count": 426, "restart_count_per_hour": 13, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 427}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.694, "sample_count": 2531, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1344470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:46:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T05:46:30+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-01T05:46:30+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-01T05:46:30.189207+09:00", "mtime_epoch": 1782852390.1892068, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160132}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:46:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5006.508, "name": "celery", "pid": 267, "rss_mb": 5006.508, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 248, "pushed_frames": 248, "read_frames": 247, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.291, "pushed": 8.291, "read": 8.258}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.373, "count": 248, "max_ms": 24.945, "p95_ms": 21.41}, "gstreamer_push": {"avg_ms": 1.218, "count": 248, "max_ms": 1.392, "p95_ms": 1.301}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.086, "count": 248, "max_ms": 45.274, "p95_ms": 42.442}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.857, "count": 248, "max_ms": 85.875, "p95_ms": 80.596}, "yolo11_640_fire_11class": {"avg_ms": 61.158, "count": 248, "max_ms": 66.347, "p95_ms": 64.908}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.751, "count": 248, "max_ms": 81.342, "p95_ms": 79.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.942, "count": 248, "max_ms": 286.938, "p95_ms": 109.242}, "read_interval": {"avg_ms": 121.034, "count": 247, "max_ms": 299.377, "p95_ms": 125.158}, "triton_wait": {"avg_ms": 76.449, "count": 248, "max_ms": 85.887, "p95_ms": 80.874}, "visualize": {"avg_ms": 3.453, "count": 248, "max_ms": 4.671, "p95_ms": 3.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11203}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10700.652, "limit_mb": 8796093022207.996}, "detail": {"pid": 81630, "restart_count": 427, "start_count": 428}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 392, "h264_decode_error_count": 1274, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:46:30+09:00", "last_error_summary": "[h264 @ 0x558e4e42f980] error while decoding MB 17 34, bytestream -5", "last_exit_at": "2026-07-01T05:46:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:46:33+09:00", "pid": 81630, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.035, "name": "ffmpeg", "pid": 81630, "rss_mb": 139.035, "state": "S (sleeping)", "threads": 106}, "restart_count": 427, "restart_count_per_hour": 14, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 428}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.694, "sample_count": 2490, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1344470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:46:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 5.011, "last_frame_pushed_at": "2026-07-01T05:46:30+09:00", "last_frame_read_age_sec": 5.119, "last_frame_read_at": "2026-07-01T05:46:30+09:00", "mjpeg": {"age_sec": 4.996, "exists": true, "mtime": "2026-07-01T05:46:30.309209+09:00", "mtime_epoch": 1782852390.3092093, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160358}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:46:35+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5006.508, "name": "celery", "pid": 267, "rss_mb": 5006.508, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.921, "pushed": 6.921, "read": 6.887}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.412, "count": 207, "max_ms": 24.945, "p95_ms": 21.493}, "gstreamer_push": {"avg_ms": 1.219, "count": 207, "max_ms": 1.392, "p95_ms": 1.302}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.133, "count": 207, "max_ms": 44.741, "p95_ms": 42.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.405, "count": 207, "max_ms": 85.875, "p95_ms": 79.709}, "yolo11_640_fire_11class": {"avg_ms": 60.872, "count": 207, "max_ms": 66.217, "p95_ms": 64.681}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.438, "count": 207, "max_ms": 81.342, "p95_ms": 79.597}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.775, "count": 207, "max_ms": 286.938, "p95_ms": 109.242}, "read_interval": {"avg_ms": 120.857, "count": 206, "max_ms": 299.377, "p95_ms": 125.158}, "triton_wait": {"avg_ms": 76.086, "count": 207, "max_ms": 85.887, "p95_ms": 80.38}, "visualize": {"avg_ms": 3.455, "count": 207, "max_ms": 4.671, "p95_ms": 3.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12383}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10530.988, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x558364ec4200] missing picture in access unit with size 18 [h264 @ 0x558364f07000] no frame! [in#0/rtsp @ 0x558364ebfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558364ebfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558364ec4840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 197, "h264_decode_error_count": 202, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:49:22+09:00", "last_error_summary": "[NULL @ 0x558364ec4200] missing picture in access unit with size 18 [h264 @ 0x558364f07000] no frame! [in#0/rtsp @ 0x558364ebfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558364ebfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558364ec4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T05:43:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 330.383, "last_started_at": "2026-07-01T05:43:51+09:00", "pid": 81478, "process": {"exists": false, "pid": 81478}, "restart_count": 196, "restart_count_per_hour": 4, "rtsp_timeout_count": 322, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 197}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.754, "sample_count": 2401, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1485961, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:49:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-01T05:49:22+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-01T05:49:22+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-01T05:49:22.236835+09:00", "mtime_epoch": 1782852562.2368345, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 188735}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:49:22+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1762.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 50, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.669, "pushed": 1.669, "read": 1.669}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.482, "count": 50, "max_ms": 20.646, "p95_ms": 20.437}, "gstreamer_push": {"avg_ms": 1.233, "count": 50, "max_ms": 1.414, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.459, "count": 50, "max_ms": 62.975, "p95_ms": 58.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.287, "count": 50, "max_ms": 85.277, "p95_ms": 80.671}, "yolo11_640_fire_11class": {"avg_ms": 52.191, "count": 50, "max_ms": 66.063, "p95_ms": 63.749}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.806, "count": 50, "max_ms": 80.541, "p95_ms": 79.222}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.561, "count": 50, "max_ms": 111.408, "p95_ms": 107.203}, "read_interval": {"avg_ms": 599.827, "count": 50, "max_ms": 20285.005, "p95_ms": 126.322}, "triton_wait": {"avg_ms": 59.912, "count": 50, "max_ms": 85.29, "p95_ms": 80.682}, "visualize": {"avg_ms": 3.437, "count": 50, "max_ms": 3.725, "p95_ms": 3.681}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12383}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10530.609, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:49:22+09:00", "last_error_summary": "[NULL @ 0x558364ec4200] missing picture in access unit with size 18 [h264 @ 0x558364f07000] no frame! [in#0/rtsp @ 0x558364ebfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558364ebfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558364ec4840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 197, "h264_decode_error_count": 202, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:49:22+09:00", "last_error_summary": "[NULL @ 0x558364ec4200] missing picture in access unit with size 18 [h264 @ 0x558364f07000] no frame! [in#0/rtsp @ 0x558364ebfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558364ebfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558364ec4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T05:49:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 330.431, "last_started_at": "2026-07-01T05:43:51+09:00", "pid": 81478, "process": {"exists": false, "pid": 81478}, "restart_count": 196, "restart_count_per_hour": 4, "rtsp_timeout_count": 322, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 197}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.754, "sample_count": 2402, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1485962, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:49:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T05:49:22+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-01T05:49:22+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-01T05:49:22.236835+09:00", "mtime_epoch": 1782852562.2368345, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 188735}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:49:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1762.758, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 50, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.667, "pushed": 1.667, "read": 1.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.483, "count": 50, "max_ms": 20.646, "p95_ms": 20.437}, "gstreamer_push": {"avg_ms": 1.236, "count": 50, "max_ms": 1.414, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.382, "count": 50, "max_ms": 62.975, "p95_ms": 58.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.455, "count": 50, "max_ms": 85.277, "p95_ms": 80.671}, "yolo11_640_fire_11class": {"avg_ms": 52.388, "count": 50, "max_ms": 66.063, "p95_ms": 63.749}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.238, "count": 50, "max_ms": 80.541, "p95_ms": 79.222}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.767, "count": 50, "max_ms": 111.408, "p95_ms": 107.203}, "read_interval": {"avg_ms": 599.827, "count": 50, "max_ms": 20285.005, "p95_ms": 126.322}, "triton_wait": {"avg_ms": 60.119, "count": 50, "max_ms": 85.29, "p95_ms": 80.682}, "visualize": {"avg_ms": 3.438, "count": 50, "max_ms": 3.725, "p95_ms": 3.681}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12383}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10530.602, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 197, "h264_decode_error_count": 202, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:49:22+09:00", "last_error_summary": "[NULL @ 0x558364ec4200] missing picture in access unit with size 18 [h264 @ 0x558364f07000] no frame! [in#0/rtsp @ 0x558364ebfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558364ebfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558364ec4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T05:49:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 330.437, "last_started_at": "2026-07-01T05:43:51+09:00", "pid": 81478, "process": {"exists": false, "pid": 81478}, "restart_count": 196, "restart_count_per_hour": 4, "rtsp_timeout_count": 322, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 197}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.754, "sample_count": 2402, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1485962, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:49:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T05:49:22+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T05:49:22+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-01T05:49:22.236835+09:00", "mtime_epoch": 1782852562.2368345, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 188735}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:49:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1762.758, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.671, "pushed": 1.671, "read": 1.637}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.483, "count": 50, "max_ms": 20.646, "p95_ms": 20.437}, "gstreamer_push": {"avg_ms": 1.236, "count": 50, "max_ms": 1.414, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.382, "count": 50, "max_ms": 62.975, "p95_ms": 58.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.455, "count": 50, "max_ms": 85.277, "p95_ms": 80.671}, "yolo11_640_fire_11class": {"avg_ms": 52.388, "count": 50, "max_ms": 66.063, "p95_ms": 63.749}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.238, "count": 50, "max_ms": 80.541, "p95_ms": 79.222}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.767, "count": 50, "max_ms": 111.408, "p95_ms": 107.203}, "read_interval": {"avg_ms": 609.944, "count": 49, "max_ms": 20285.005, "p95_ms": 126.322}, "triton_wait": {"avg_ms": 60.119, "count": 50, "max_ms": 85.29, "p95_ms": 80.682}, "visualize": {"avg_ms": 3.438, "count": 50, "max_ms": 3.725, "p95_ms": 3.681}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12383}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10709.094, "limit_mb": 8796093022207.996}, "detail": {"pid": 81737, "restart_count": 197, "start_count": 198}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 197, "h264_decode_error_count": 202, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:49:22+09:00", "last_error_summary": "[NULL @ 0x558364ec4200] missing picture in access unit with size 18 [h264 @ 0x558364f07000] no frame! [in#0/rtsp @ 0x558364ebfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558364ebfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x558364ec4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T05:49:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:49:24+09:00", "pid": 81737, "process": {"exists": true, "fd_count": 5, "hwm_mb": 207.348, "name": "ffmpeg", "pid": 81737, "rss_mb": 207.348, "state": "S (sleeping)", "threads": 106}, "restart_count": 197, "restart_count_per_hour": 5, "rtsp_timeout_count": 322, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 198}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.77, "sample_count": 2368, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1485962, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:49:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.177, "last_frame_pushed_at": "2026-07-01T05:49:22+09:00", "last_frame_read_age_sec": 4.278, "last_frame_read_at": "2026-07-01T05:49:22+09:00", "mjpeg": {"age_sec": 4.16, "exists": true, "mtime": "2026-07-01T05:49:22.352837+09:00", "mtime_epoch": 1782852562.3528368, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 191117}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T05:49:26+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1762.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.506, "pushed": 0.506, "read": 0.506}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.527, "count": 13, "max_ms": 20.646, "p95_ms": 20.646}, "gstreamer_push": {"avg_ms": 1.23, "count": 13, "max_ms": 1.32, "p95_ms": 1.32}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.695, "count": 13, "max_ms": 62.61, "p95_ms": 62.61}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.809, "count": 13, "max_ms": 79.804, "p95_ms": 79.804}, "yolo11_640_fire_11class": {"avg_ms": 53.517, "count": 13, "max_ms": 65.714, "p95_ms": 65.714}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.932, "count": 13, "max_ms": 80.541, "p95_ms": 80.541}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.215, "count": 13, "max_ms": 106.639, "p95_ms": 106.639}, "read_interval": {"avg_ms": 2017.255, "count": 13, "max_ms": 20285.005, "p95_ms": 20285.005}, "triton_wait": {"avg_ms": 61.616, "count": 13, "max_ms": 80.553, "p95_ms": 80.553}, "visualize": {"avg_ms": 3.366, "count": 13, "max_ms": 3.473, "p95_ms": 3.473}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 25.684, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10598.922, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55783c6d9380] error while decoding MB 43 17, bytestream -34", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 393, "h264_decode_error_count": 1275, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:50:10+09:00", "last_error_summary": "[h264 @ 0x55783c6d9380] error while decoding MB 43 17, bytestream -34", "last_exit_at": "2026-07-01T05:46:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 217.407, "last_started_at": "2026-07-01T05:46:33+09:00", "pid": 81630, "process": {"exists": false, "pid": 81630}, "restart_count": 427, "restart_count_per_hour": 14, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 428}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.492, "sample_count": 2263, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1346075, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:50:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T05:50:10+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T05:50:10+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T05:50:10.665854+09:00", "mtime_epoch": 1782852610.6658535, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163416}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:50:10+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5007.281, "name": "celery", "pid": 267, "rss_mb": 5007.281, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.021, "pushed": 6.021, "read": 6.021}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.338, "count": 180, "max_ms": 32.492, "p95_ms": 28.909}, "gstreamer_push": {"avg_ms": 1.01, "count": 180, "max_ms": 1.566, "p95_ms": 1.144}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.173, "count": 180, "max_ms": 41.95, "p95_ms": 34.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.246, "count": 180, "max_ms": 85.868, "p95_ms": 79.91}, "yolo11_640_fire_11class": {"avg_ms": 47.804, "count": 180, "max_ms": 65.282, "p95_ms": 61.762}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.777, "count": 180, "max_ms": 79.75, "p95_ms": 77.29}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.285, "count": 180, "max_ms": 202.305, "p95_ms": 176.752}, "read_interval": {"avg_ms": 166.577, "count": 180, "max_ms": 212.243, "p95_ms": 187.942}, "triton_wait": {"avg_ms": 63.292, "count": 180, "max_ms": 85.881, "p95_ms": 79.922}, "visualize": {"avg_ms": 4.77, "count": 180, "max_ms": 6.085, "p95_ms": 5.638}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10598.27, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:50:10+09:00", "last_error_summary": "[h264 @ 0x55783c6d9380] error while decoding MB 43 17, bytestream -34", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 393, "h264_decode_error_count": 1275, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:50:10+09:00", "last_error_summary": "[h264 @ 0x55783c6d9380] error while decoding MB 43 17, bytestream -34", "last_exit_at": "2026-07-01T05:50:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 217.492, "last_started_at": "2026-07-01T05:46:33+09:00", "pid": 81630, "process": {"exists": false, "pid": 81630}, "restart_count": 427, "restart_count_per_hour": 14, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 428}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.492, "sample_count": 2263, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1346076, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:50:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T05:50:10+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-01T05:50:10+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T05:50:10.665854+09:00", "mtime_epoch": 1782852610.6658535, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163416}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:50:10+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5007.281, "name": "celery", "pid": 267, "rss_mb": 5007.281, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.037, "pushed": 6.037, "read": 6.004}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.35, "count": 180, "max_ms": 32.492, "p95_ms": 28.909}, "gstreamer_push": {"avg_ms": 1.01, "count": 181, "max_ms": 1.566, "p95_ms": 1.144}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.188, "count": 180, "max_ms": 41.95, "p95_ms": 34.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.206, "count": 180, "max_ms": 85.868, "p95_ms": 79.91}, "yolo11_640_fire_11class": {"avg_ms": 47.754, "count": 180, "max_ms": 65.282, "p95_ms": 61.762}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.675, "count": 180, "max_ms": 79.75, "p95_ms": 77.29}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.147, "count": 181, "max_ms": 202.305, "p95_ms": 176.752}, "read_interval": {"avg_ms": 166.577, "count": 180, "max_ms": 212.243, "p95_ms": 187.942}, "triton_wait": {"avg_ms": 63.232, "count": 180, "max_ms": 85.881, "p95_ms": 79.922}, "visualize": {"avg_ms": 4.77, "count": 180, "max_ms": 6.085, "p95_ms": 5.638}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10598.242, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 393, "h264_decode_error_count": 1275, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:50:10+09:00", "last_error_summary": "[h264 @ 0x55783c6d9380] error while decoding MB 43 17, bytestream -34", "last_exit_at": "2026-07-01T05:50:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 217.497, "last_started_at": "2026-07-01T05:46:33+09:00", "pid": 81630, "process": {"exists": false, "pid": 81630}, "restart_count": 427, "restart_count_per_hour": 14, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 428}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.492, "sample_count": 2263, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1346076, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:50:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T05:50:10+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-01T05:50:10+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T05:50:10.665854+09:00", "mtime_epoch": 1782852610.6658535, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163416}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:50:10+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5007.281, "name": "celery", "pid": 267, "rss_mb": 5007.281, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.036, "pushed": 6.036, "read": 6.003}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.35, "count": 180, "max_ms": 32.492, "p95_ms": 28.909}, "gstreamer_push": {"avg_ms": 1.01, "count": 181, "max_ms": 1.566, "p95_ms": 1.144}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.188, "count": 180, "max_ms": 41.95, "p95_ms": 34.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.206, "count": 180, "max_ms": 85.868, "p95_ms": 79.91}, "yolo11_640_fire_11class": {"avg_ms": 47.754, "count": 180, "max_ms": 65.282, "p95_ms": 61.762}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.675, "count": 180, "max_ms": 79.75, "p95_ms": 77.29}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.147, "count": 181, "max_ms": 202.305, "p95_ms": 176.752}, "read_interval": {"avg_ms": 166.577, "count": 180, "max_ms": 212.243, "p95_ms": 187.942}, "triton_wait": {"avg_ms": 63.232, "count": 180, "max_ms": 85.881, "p95_ms": 79.922}, "visualize": {"avg_ms": 4.77, "count": 180, "max_ms": 6.085, "p95_ms": 5.638}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10706.934, "limit_mb": 8796093022207.996}, "detail": {"pid": 81889, "restart_count": 428, "start_count": 429}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 393, "h264_decode_error_count": 1275, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:50:10+09:00", "last_error_summary": "[h264 @ 0x55783c6d9380] error while decoding MB 43 17, bytestream -34", "last_exit_at": "2026-07-01T05:50:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:50:13+09:00", "pid": 81889, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.375, "name": "ffmpeg", "pid": 81889, "rss_mb": 139.375, "state": "S (sleeping)", "threads": 106}, "restart_count": 428, "restart_count_per_hour": 15, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 429}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.657, "sample_count": 2225, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1346076, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:50:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.699, "last_frame_pushed_at": "2026-07-01T05:50:10+09:00", "last_frame_read_age_sec": 4.831, "last_frame_read_at": "2026-07-01T05:50:10+09:00", "mjpeg": {"age_sec": 4.681, "exists": true, "mtime": "2026-07-01T05:50:10.809856+09:00", "mtime_epoch": 1782852610.8098564, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163547}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:50:15+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5007.281, "name": "celery", "pid": 267, "rss_mb": 5007.281, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.122, "pushed": 5.122, "read": 5.088}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.295, "count": 153, "max_ms": 32.492, "p95_ms": 28.909}, "gstreamer_push": {"avg_ms": 1.012, "count": 153, "max_ms": 1.566, "p95_ms": 1.144}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.353, "count": 153, "max_ms": 41.95, "p95_ms": 34.642}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.007, "count": 153, "max_ms": 85.614, "p95_ms": 79.91}, "yolo11_640_fire_11class": {"avg_ms": 47.637, "count": 153, "max_ms": 65.282, "p95_ms": 61.762}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.58, "count": 153, "max_ms": 79.75, "p95_ms": 77.29}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.373, "count": 153, "max_ms": 202.305, "p95_ms": 176.752}, "read_interval": {"avg_ms": 165.818, "count": 152, "max_ms": 212.243, "p95_ms": 187.942}, "triton_wait": {"avg_ms": 63.036, "count": 153, "max_ms": 85.626, "p95_ms": 79.922}, "visualize": {"avg_ms": 4.752, "count": 153, "max_ms": 6.085, "p95_ms": 5.628}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11228}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10601.746, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e711417f40] error while decoding MB 55 6, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 394, "h264_decode_error_count": 1276, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:53:23+09:00", "last_error_summary": "[h264 @ 0x55e711417f40] error while decoding MB 55 6, bytestream -7", "last_exit_at": "2026-07-01T05:50:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 189.938, "last_started_at": "2026-07-01T05:50:13+09:00", "pid": 81889, "process": {"exists": false, "pid": 81889}, "restart_count": 428, "restart_count_per_hour": 15, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 429}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.116, "sample_count": 2137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1347439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:53:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T05:53:23+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T05:53:23+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T05:53:23.381901+09:00", "mtime_epoch": 1782852803.3819005, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162018}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:53:23+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5007.797, "name": "celery", "pid": 267, "rss_mb": 5007.797, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 220, "pushed_frames": 220, "read_frames": 220, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.347, "pushed": 7.347, "read": 7.347}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.601, "count": 220, "max_ms": 32.151, "p95_ms": 25.684}, "gstreamer_push": {"avg_ms": 1.161, "count": 220, "max_ms": 1.962, "p95_ms": 1.28}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.289, "count": 220, "max_ms": 45.48, "p95_ms": 40.91}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.114, "count": 220, "max_ms": 91.686, "p95_ms": 83.537}, "yolo11_640_fire_11class": {"avg_ms": 54.077, "count": 220, "max_ms": 69.473, "p95_ms": 66.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.07, "count": 220, "max_ms": 84.695, "p95_ms": 80.481}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.173, "count": 220, "max_ms": 174.737, "p95_ms": 149.495}, "read_interval": {"avg_ms": 136.44, "count": 220, "max_ms": 186.039, "p95_ms": 162.291}, "triton_wait": {"avg_ms": 70.331, "count": 220, "max_ms": 91.698, "p95_ms": 83.549}, "visualize": {"avg_ms": 4.224, "count": 220, "max_ms": 5.92, "p95_ms": 5.486}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11228}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10601.078, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:53:23+09:00", "last_error_summary": "[h264 @ 0x55e711417f40] error while decoding MB 55 6, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 394, "h264_decode_error_count": 1276, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:53:23+09:00", "last_error_summary": "[h264 @ 0x55e711417f40] error while decoding MB 55 6, bytestream -7", "last_exit_at": "2026-07-01T05:53:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.045, "last_started_at": "2026-07-01T05:50:13+09:00", "pid": 81889, "process": {"exists": false, "pid": 81889}, "restart_count": 428, "restart_count_per_hour": 15, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 429}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.116, "sample_count": 2137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1347440, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:53:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T05:53:23+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-01T05:53:23+09:00", "mjpeg": {"age_sec": 0.153, "exists": true, "mtime": "2026-07-01T05:53:23.381901+09:00", "mtime_epoch": 1782852803.3819005, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162018}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:53:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5007.797, "name": "celery", "pid": 267, "rss_mb": 5007.797, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 220, "pushed_frames": 220, "read_frames": 219, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.366, "pushed": 7.366, "read": 7.333}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.601, "count": 220, "max_ms": 32.151, "p95_ms": 25.684}, "gstreamer_push": {"avg_ms": 1.161, "count": 220, "max_ms": 1.962, "p95_ms": 1.28}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.308, "count": 220, "max_ms": 45.48, "p95_ms": 40.91}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.24, "count": 220, "max_ms": 91.686, "p95_ms": 83.59}, "yolo11_640_fire_11class": {"avg_ms": 54.151, "count": 220, "max_ms": 69.473, "p95_ms": 66.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.17, "count": 220, "max_ms": 84.695, "p95_ms": 80.481}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.298, "count": 220, "max_ms": 174.737, "p95_ms": 149.803}, "read_interval": {"avg_ms": 136.444, "count": 219, "max_ms": 186.039, "p95_ms": 162.506}, "triton_wait": {"avg_ms": 70.458, "count": 220, "max_ms": 91.698, "p95_ms": 83.601}, "visualize": {"avg_ms": 4.221, "count": 220, "max_ms": 5.92, "p95_ms": 5.486}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11228}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10601.195, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 394, "h264_decode_error_count": 1276, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:53:23+09:00", "last_error_summary": "[h264 @ 0x55e711417f40] error while decoding MB 55 6, bytestream -7", "last_exit_at": "2026-07-01T05:53:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 190.051, "last_started_at": "2026-07-01T05:50:13+09:00", "pid": 81889, "process": {"exists": false, "pid": 81889}, "restart_count": 428, "restart_count_per_hour": 15, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 429}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.116, "sample_count": 2137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1347440, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:53:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T05:53:23+09:00", "last_frame_read_age_sec": 0.165, "last_frame_read_at": "2026-07-01T05:53:23+09:00", "mjpeg": {"age_sec": 0.158, "exists": true, "mtime": "2026-07-01T05:53:23.381901+09:00", "mtime_epoch": 1782852803.3819005, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162018}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:53:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5007.797, "name": "celery", "pid": 267, "rss_mb": 5007.797, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 220, "pushed_frames": 220, "read_frames": 219, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.365, "pushed": 7.365, "read": 7.332}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.601, "count": 220, "max_ms": 32.151, "p95_ms": 25.684}, "gstreamer_push": {"avg_ms": 1.161, "count": 220, "max_ms": 1.962, "p95_ms": 1.28}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.308, "count": 220, "max_ms": 45.48, "p95_ms": 40.91}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.24, "count": 220, "max_ms": 91.686, "p95_ms": 83.59}, "yolo11_640_fire_11class": {"avg_ms": 54.151, "count": 220, "max_ms": 69.473, "p95_ms": 66.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.17, "count": 220, "max_ms": 84.695, "p95_ms": 80.481}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.298, "count": 220, "max_ms": 174.737, "p95_ms": 149.803}, "read_interval": {"avg_ms": 136.444, "count": 219, "max_ms": 186.039, "p95_ms": 162.506}, "triton_wait": {"avg_ms": 70.458, "count": 220, "max_ms": 91.698, "p95_ms": 83.601}, "visualize": {"avg_ms": 4.221, "count": 220, "max_ms": 5.92, "p95_ms": 5.486}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11228}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10709.273, "limit_mb": 8796093022207.996}, "detail": {"pid": 81996, "restart_count": 429, "start_count": 430}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 394, "h264_decode_error_count": 1276, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:53:23+09:00", "last_error_summary": "[h264 @ 0x55e711417f40] error while decoding MB 55 6, bytestream -7", "last_exit_at": "2026-07-01T05:53:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:53:26+09:00", "pid": 81996, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.227, "name": "ffmpeg", "pid": 81996, "rss_mb": 139.227, "state": "S (sleeping)", "threads": 106}, "restart_count": 429, "restart_count_per_hour": 16, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 430}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 164.308, "sample_count": 2100, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1347440, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:53:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.555, "last_frame_pushed_at": "2026-07-01T05:53:23+09:00", "last_frame_read_age_sec": 4.707, "last_frame_read_at": "2026-07-01T05:53:23+09:00", "mjpeg": {"age_sec": 4.537, "exists": true, "mtime": "2026-07-01T05:53:23.545904+09:00", "mtime_epoch": 1782852803.545904, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156348}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:53:28+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5007.797, "name": "celery", "pid": 267, "rss_mb": 5007.797, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.318, "pushed": 6.318, "read": 6.285}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.442, "count": 189, "max_ms": 32.151, "p95_ms": 25.684}, "gstreamer_push": {"avg_ms": 1.172, "count": 189, "max_ms": 1.962, "p95_ms": 1.289}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.639, "count": 189, "max_ms": 45.48, "p95_ms": 41.255}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.098, "count": 189, "max_ms": 91.686, "p95_ms": 83.933}, "yolo11_640_fire_11class": {"avg_ms": 54.707, "count": 189, "max_ms": 69.473, "p95_ms": 66.749}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.598, "count": 189, "max_ms": 84.695, "p95_ms": 80.534}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.567, "count": 189, "max_ms": 168.955, "p95_ms": 148.651}, "read_interval": {"avg_ms": 134.772, "count": 188, "max_ms": 180.402, "p95_ms": 161.245}, "triton_wait": {"avg_ms": 71.251, "count": 189, "max_ms": 91.698, "p95_ms": 84.707}, "visualize": {"avg_ms": 4.135, "count": 189, "max_ms": 5.638, "p95_ms": 5.123}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10603.594, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5637501ea780] error while decoding MB 41 25, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 395, "h264_decode_error_count": 1277, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:56:09+09:00", "last_error_summary": "[h264 @ 0x5637501ea780] error while decoding MB 41 25, bytestream -5", "last_exit_at": "2026-07-01T05:53:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.721, "last_started_at": "2026-07-01T05:53:26+09:00", "pid": 81996, "process": {"exists": false, "pid": 81996}, "restart_count": 429, "restart_count_per_hour": 16, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 430}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.245, "sample_count": 2091, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1348526, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:56:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T05:56:09+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T05:56:09+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T05:56:09.757387+09:00", "mtime_epoch": 1782852969.7573874, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160307}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:56:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5008.312, "name": "celery", "pid": 267, "rss_mb": 5008.312, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 196, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.539, "pushed": 6.539, "read": 6.539}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.039, "count": 195, "max_ms": 39.287, "p95_ms": 28.116}, "gstreamer_push": {"avg_ms": 1.088, "count": 196, "max_ms": 1.55, "p95_ms": 1.214}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.904, "count": 195, "max_ms": 45.745, "p95_ms": 36.257}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.164, "count": 195, "max_ms": 95.583, "p95_ms": 87.669}, "yolo11_640_fire_11class": {"avg_ms": 48.704, "count": 195, "max_ms": 68.319, "p95_ms": 63.581}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.701, "count": 195, "max_ms": 83.35, "p95_ms": 77.997}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.275, "count": 196, "max_ms": 180.358, "p95_ms": 167.669}, "read_interval": {"avg_ms": 153.397, "count": 196, "max_ms": 191.419, "p95_ms": 180.227}, "triton_wait": {"avg_ms": 68.181, "count": 195, "max_ms": 95.595, "p95_ms": 87.68}, "visualize": {"avg_ms": 4.871, "count": 195, "max_ms": 6.095, "p95_ms": 5.781}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10602.664, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:56:09+09:00", "last_error_summary": "[h264 @ 0x5637501ea780] error while decoding MB 41 25, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 395, "h264_decode_error_count": 1277, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:56:09+09:00", "last_error_summary": "[h264 @ 0x5637501ea780] error while decoding MB 41 25, bytestream -5", "last_exit_at": "2026-07-01T05:56:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.848, "last_started_at": "2026-07-01T05:53:26+09:00", "pid": 81996, "process": {"exists": false, "pid": 81996}, "restart_count": 429, "restart_count_per_hour": 16, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 430}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.468, "sample_count": 2091, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1348527, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:56:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T05:56:09+09:00", "last_frame_read_age_sec": 0.179, "last_frame_read_at": "2026-07-01T05:56:09+09:00", "mjpeg": {"age_sec": 0.173, "exists": true, "mtime": "2026-07-01T05:56:09.757387+09:00", "mtime_epoch": 1782852969.7573874, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160307}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:56:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5008.312, "name": "celery", "pid": 267, "rss_mb": 5008.312, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.548, "pushed": 6.548, "read": 6.515}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.036, "count": 196, "max_ms": 39.287, "p95_ms": 28.116}, "gstreamer_push": {"avg_ms": 1.087, "count": 196, "max_ms": 1.55, "p95_ms": 1.214}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.896, "count": 196, "max_ms": 45.745, "p95_ms": 36.257}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.254, "count": 196, "max_ms": 95.583, "p95_ms": 87.669}, "yolo11_640_fire_11class": {"avg_ms": 48.782, "count": 196, "max_ms": 68.319, "p95_ms": 63.657}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.806, "count": 196, "max_ms": 83.35, "p95_ms": 78.102}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.418, "count": 196, "max_ms": 180.358, "p95_ms": 168.383}, "read_interval": {"avg_ms": 153.377, "count": 195, "max_ms": 191.419, "p95_ms": 180.227}, "triton_wait": {"avg_ms": 68.27, "count": 196, "max_ms": 95.595, "p95_ms": 87.68}, "visualize": {"avg_ms": 4.874, "count": 196, "max_ms": 6.095, "p95_ms": 5.781}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10602.656, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 395, "h264_decode_error_count": 1277, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:56:09+09:00", "last_error_summary": "[h264 @ 0x5637501ea780] error while decoding MB 41 25, bytestream -5", "last_exit_at": "2026-07-01T05:56:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.852, "last_started_at": "2026-07-01T05:53:26+09:00", "pid": 81996, "process": {"exists": false, "pid": 81996}, "restart_count": 429, "restart_count_per_hour": 16, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 430}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.468, "sample_count": 2091, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1348527, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:56:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T05:56:09+09:00", "last_frame_read_age_sec": 0.183, "last_frame_read_at": "2026-07-01T05:56:09+09:00", "mjpeg": {"age_sec": 0.177, "exists": true, "mtime": "2026-07-01T05:56:09.757387+09:00", "mtime_epoch": 1782852969.7573874, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160307}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:56:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5008.312, "name": "celery", "pid": 267, "rss_mb": 5008.312, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.547, "pushed": 6.547, "read": 6.514}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.036, "count": 196, "max_ms": 39.287, "p95_ms": 28.116}, "gstreamer_push": {"avg_ms": 1.087, "count": 196, "max_ms": 1.55, "p95_ms": 1.214}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.896, "count": 196, "max_ms": 45.745, "p95_ms": 36.257}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.254, "count": 196, "max_ms": 95.583, "p95_ms": 87.669}, "yolo11_640_fire_11class": {"avg_ms": 48.782, "count": 196, "max_ms": 68.319, "p95_ms": 63.657}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.806, "count": 196, "max_ms": 83.35, "p95_ms": 78.102}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.418, "count": 196, "max_ms": 180.358, "p95_ms": 168.383}, "read_interval": {"avg_ms": 153.377, "count": 195, "max_ms": 191.419, "p95_ms": 180.227}, "triton_wait": {"avg_ms": 68.27, "count": 196, "max_ms": 95.595, "p95_ms": 87.68}, "visualize": {"avg_ms": 4.874, "count": 196, "max_ms": 6.095, "p95_ms": 5.781}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10710.914, "limit_mb": 8796093022207.996}, "detail": {"pid": 82103, "restart_count": 430, "start_count": 431}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 395, "h264_decode_error_count": 1277, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:56:09+09:00", "last_error_summary": "[h264 @ 0x5637501ea780] error while decoding MB 41 25, bytestream -5", "last_exit_at": "2026-07-01T05:56:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:56:12+09:00", "pid": 82103, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.914, "name": "ffmpeg", "pid": 82103, "rss_mb": 138.914, "state": "S (sleeping)", "threads": 106}, "restart_count": 430, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 431}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.633, "sample_count": 2068, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1348527, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:56:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.145, "last_frame_pushed_at": "2026-07-01T05:56:09+09:00", "last_frame_read_age_sec": 4.319, "last_frame_read_at": "2026-07-01T05:56:09+09:00", "mjpeg": {"age_sec": 4.129, "exists": true, "mtime": "2026-07-01T05:56:09.941391+09:00", "mtime_epoch": 1782852969.9413912, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159189}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:56:14+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5008.312, "name": "celery", "pid": 267, "rss_mb": 5008.312, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 171, "pushed_frames": 171, "read_frames": 170, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.714, "pushed": 5.714, "read": 5.68}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.92, "count": 171, "max_ms": 39.287, "p95_ms": 28.021}, "gstreamer_push": {"avg_ms": 1.095, "count": 171, "max_ms": 1.55, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.842, "count": 171, "max_ms": 45.745, "p95_ms": 35.898}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.631, "count": 171, "max_ms": 95.583, "p95_ms": 87.669}, "yolo11_640_fire_11class": {"avg_ms": 48.996, "count": 171, "max_ms": 68.319, "p95_ms": 63.976}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.902, "count": 171, "max_ms": 83.35, "p95_ms": 78.102}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.598, "count": 171, "max_ms": 179.895, "p95_ms": 166.943}, "read_interval": {"avg_ms": 151.577, "count": 170, "max_ms": 190.664, "p95_ms": 176.572}, "triton_wait": {"avg_ms": 68.648, "count": 171, "max_ms": 95.595, "p95_ms": 87.68}, "visualize": {"avg_ms": 4.783, "count": 171, "max_ms": 6.095, "p95_ms": 5.745}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11244}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10605.938, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5651c459f580] error while decoding MB 25 25, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 396, "h264_decode_error_count": 1278, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:58:23+09:00", "last_error_summary": "[h264 @ 0x5651c459f580] error while decoding MB 25 25, bytestream -5", "last_exit_at": "2026-07-01T05:56:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.871, "last_started_at": "2026-07-01T05:56:12+09:00", "pid": 82103, "process": {"exists": false, "pid": 82103}, "restart_count": 430, "restart_count_per_hour": 16, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 431}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.771, "sample_count": 1854, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:58:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T05:58:23+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T05:58:23+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T05:58:23.892195+09:00", "mtime_epoch": 1782853103.8921947, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162306}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:58:23+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5008.828, "name": "celery", "pid": 267, "rss_mb": 5008.828, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.834, "pushed": 5.834, "read": 5.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.553, "count": 175, "max_ms": 31.457, "p95_ms": 28.665}, "gstreamer_push": {"avg_ms": 1.031, "count": 175, "max_ms": 1.634, "p95_ms": 1.13}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.204, "count": 175, "max_ms": 40.286, "p95_ms": 36.594}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.366, "count": 175, "max_ms": 94.131, "p95_ms": 84.946}, "yolo11_640_fire_11class": {"avg_ms": 48.58, "count": 175, "max_ms": 64.41, "p95_ms": 62.279}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.824, "count": 175, "max_ms": 85.103, "p95_ms": 80.538}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.011, "count": 175, "max_ms": 184.806, "p95_ms": 177.658}, "read_interval": {"avg_ms": 171.064, "count": 176, "max_ms": 195.752, "p95_ms": 188.493}, "triton_wait": {"avg_ms": 69.397, "count": 175, "max_ms": 94.143, "p95_ms": 84.958}, "visualize": {"avg_ms": 5.635, "count": 175, "max_ms": 6.696, "p95_ms": 6.226}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11244}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10605.309, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T05:58:23+09:00", "last_error_summary": "[h264 @ 0x5651c459f580] error while decoding MB 25 25, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 396, "h264_decode_error_count": 1278, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:58:23+09:00", "last_error_summary": "[h264 @ 0x5651c459f580] error while decoding MB 25 25, bytestream -5", "last_exit_at": "2026-07-01T05:58:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.007, "last_started_at": "2026-07-01T05:56:12+09:00", "pid": 82103, "process": {"exists": false, "pid": 82103}, "restart_count": 430, "restart_count_per_hour": 16, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 431}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.836, "sample_count": 1855, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:58:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T05:58:24+09:00", "last_frame_read_age_sec": 0.189, "last_frame_read_at": "2026-07-01T05:58:23+09:00", "mjpeg": {"age_sec": 0.185, "exists": true, "mtime": "2026-07-01T05:58:23.892195+09:00", "mtime_epoch": 1782853103.8921947, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162306}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:58:24+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5008.828, "name": "celery", "pid": 267, "rss_mb": 5008.828, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 176, "pushed_frames": 176, "read_frames": 175, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.872, "pushed": 5.872, "read": 5.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.508, "count": 175, "max_ms": 31.309, "p95_ms": 28.657}, "gstreamer_push": {"avg_ms": 1.031, "count": 176, "max_ms": 1.634, "p95_ms": 1.13}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.179, "count": 175, "max_ms": 40.286, "p95_ms": 36.594}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.538, "count": 175, "max_ms": 94.131, "p95_ms": 85.291}, "yolo11_640_fire_11class": {"avg_ms": 48.697, "count": 175, "max_ms": 64.621, "p95_ms": 62.781}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.991, "count": 175, "max_ms": 85.103, "p95_ms": 80.674}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.135, "count": 176, "max_ms": 184.806, "p95_ms": 178.992}, "read_interval": {"avg_ms": 171.093, "count": 175, "max_ms": 195.752, "p95_ms": 188.493}, "triton_wait": {"avg_ms": 69.569, "count": 175, "max_ms": 94.143, "p95_ms": 85.303}, "visualize": {"avg_ms": 5.638, "count": 175, "max_ms": 6.696, "p95_ms": 6.226}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11244}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10605.301, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 396, "h264_decode_error_count": 1278, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:58:23+09:00", "last_error_summary": "[h264 @ 0x5651c459f580] error while decoding MB 25 25, bytestream -5", "last_exit_at": "2026-07-01T05:58:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.011, "last_started_at": "2026-07-01T05:56:12+09:00", "pid": 82103, "process": {"exists": false, "pid": 82103}, "restart_count": 430, "restart_count_per_hour": 16, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 431}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.836, "sample_count": 1855, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:58:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T05:58:24+09:00", "last_frame_read_age_sec": 0.193, "last_frame_read_at": "2026-07-01T05:58:23+09:00", "mjpeg": {"age_sec": 0.188, "exists": true, "mtime": "2026-07-01T05:58:23.892195+09:00", "mtime_epoch": 1782853103.8921947, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162306}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:58:24+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5008.828, "name": "celery", "pid": 267, "rss_mb": 5008.828, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 176, "pushed_frames": 176, "read_frames": 175, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.871, "pushed": 5.871, "read": 5.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.508, "count": 175, "max_ms": 31.309, "p95_ms": 28.657}, "gstreamer_push": {"avg_ms": 1.031, "count": 176, "max_ms": 1.634, "p95_ms": 1.13}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.179, "count": 175, "max_ms": 40.286, "p95_ms": 36.594}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.538, "count": 175, "max_ms": 94.131, "p95_ms": 85.291}, "yolo11_640_fire_11class": {"avg_ms": 48.697, "count": 175, "max_ms": 64.621, "p95_ms": 62.781}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.991, "count": 175, "max_ms": 85.103, "p95_ms": 80.674}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.135, "count": 176, "max_ms": 184.806, "p95_ms": 178.992}, "read_interval": {"avg_ms": 171.093, "count": 175, "max_ms": 195.752, "p95_ms": 188.493}, "triton_wait": {"avg_ms": 69.569, "count": 175, "max_ms": 94.143, "p95_ms": 85.303}, "visualize": {"avg_ms": 5.638, "count": 175, "max_ms": 6.696, "p95_ms": 6.226}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11244}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10713.465, "limit_mb": 8796093022207.996}, "detail": {"pid": 82210, "restart_count": 431, "start_count": 432}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 396, "h264_decode_error_count": 1278, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T05:58:23+09:00", "last_error_summary": "[h264 @ 0x5651c459f580] error while decoding MB 25 25, bytestream -5", "last_exit_at": "2026-07-01T05:58:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T05:58:26+09:00", "pid": 82210, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.211, "name": "ffmpeg", "pid": 82210, "rss_mb": 139.211, "state": "S (sleeping)", "threads": 106}, "restart_count": 431, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 432}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.875, "sample_count": 1850, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T05:58:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.703, "last_frame_pushed_at": "2026-07-01T05:58:24+09:00", "last_frame_read_age_sec": 4.886, "last_frame_read_at": "2026-07-01T05:58:23+09:00", "mjpeg": {"age_sec": 4.69, "exists": true, "mtime": "2026-07-01T05:58:24.084199+09:00", "mtime_epoch": 1782853104.0841987, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161733}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T05:58:28+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5008.828, "name": "celery", "pid": 267, "rss_mb": 5008.828, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.959, "pushed": 4.959, "read": 4.926}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.407, "count": 148, "max_ms": 31.309, "p95_ms": 28.611}, "gstreamer_push": {"avg_ms": 1.028, "count": 148, "max_ms": 1.634, "p95_ms": 1.115}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.323, "count": 148, "max_ms": 40.286, "p95_ms": 35.918}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.528, "count": 148, "max_ms": 94.131, "p95_ms": 84.946}, "yolo11_640_fire_11class": {"avg_ms": 48.611, "count": 148, "max_ms": 64.621, "p95_ms": 62.781}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.738, "count": 148, "max_ms": 83.25, "p95_ms": 80.538}}, "truncated_count": 0}, "processing_total": {"avg_ms": 159.948, "count": 148, "max_ms": 184.806, "p95_ms": 177.658}, "read_interval": {"avg_ms": 170.866, "count": 147, "max_ms": 195.752, "p95_ms": 187.936}, "triton_wait": {"avg_ms": 69.562, "count": 148, "max_ms": 94.143, "p95_ms": 84.958}, "visualize": {"avg_ms": 5.614, "count": 148, "max_ms": 6.696, "p95_ms": 6.196}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.842, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11249}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10606.539, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ba42a17d40] error while decoding MB 54 16, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 397, "h264_decode_error_count": 1279, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:00:27+09:00", "last_error_summary": "[h264 @ 0x55ba42a17d40] error while decoding MB 54 16, bytestream -7", "last_exit_at": "2026-07-01T05:58:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.186, "last_started_at": "2026-07-01T05:58:26+09:00", "pid": 82210, "process": {"exists": false, "pid": 82210}, "restart_count": 431, "restart_count_per_hour": 16, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 432}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.423, "sample_count": 1647, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349902, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:00:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T06:00:27+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T06:00:27+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T06:00:27.918788+09:00", "mtime_epoch": 1782853227.9187884, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166469}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:00:27+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5009.086, "name": "celery", "pid": 267, "rss_mb": 5009.086, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 161, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.386, "pushed": 5.386, "read": 5.386}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.317, "count": 161, "max_ms": 32.524, "p95_ms": 28.601}, "gstreamer_push": {"avg_ms": 0.94, "count": 161, "max_ms": 1.122, "p95_ms": 1.077}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.597, "count": 161, "max_ms": 45.82, "p95_ms": 34.535}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.238, "count": 161, "max_ms": 92.262, "p95_ms": 85.357}, "yolo11_640_fire_11class": {"avg_ms": 46.284, "count": 161, "max_ms": 64.64, "p95_ms": 61.446}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.067, "count": 161, "max_ms": 89.234, "p95_ms": 82.218}}, "truncated_count": 0}, "processing_total": {"avg_ms": 175.811, "count": 161, "max_ms": 237.274, "p95_ms": 218.328}, "read_interval": {"avg_ms": 186.311, "count": 161, "max_ms": 248.203, "p95_ms": 228.094}, "triton_wait": {"avg_ms": 67.265, "count": 161, "max_ms": 92.274, "p95_ms": 85.37}, "visualize": {"avg_ms": 6.47, "count": 161, "max_ms": 11.345, "p95_ms": 8.4}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11249}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10605.867, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:00:27+09:00", "last_error_summary": "[h264 @ 0x55ba42a17d40] error while decoding MB 54 16, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 397, "h264_decode_error_count": 1279, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:00:27+09:00", "last_error_summary": "[h264 @ 0x55ba42a17d40] error while decoding MB 54 16, bytestream -7", "last_exit_at": "2026-07-01T06:00:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.319, "last_started_at": "2026-07-01T05:58:26+09:00", "pid": 82210, "process": {"exists": false, "pid": 82210}, "restart_count": 431, "restart_count_per_hour": 16, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 432}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.423, "sample_count": 1647, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:00:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:00:28+09:00", "last_frame_read_age_sec": 0.183, "last_frame_read_at": "2026-07-01T06:00:27+09:00", "mjpeg": {"age_sec": 0.174, "exists": true, "mtime": "2026-07-01T06:00:27.918788+09:00", "mtime_epoch": 1782853227.9187884, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166469}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:00:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5009.086, "name": "celery", "pid": 267, "rss_mb": 5009.086, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.396, "pushed": 5.396, "read": 5.362}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.324, "count": 161, "max_ms": 32.524, "p95_ms": 28.601}, "gstreamer_push": {"avg_ms": 0.939, "count": 161, "max_ms": 1.122, "p95_ms": 1.071}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.596, "count": 161, "max_ms": 45.82, "p95_ms": 34.535}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.278, "count": 161, "max_ms": 92.262, "p95_ms": 85.357}, "yolo11_640_fire_11class": {"avg_ms": 46.343, "count": 161, "max_ms": 64.64, "p95_ms": 61.446}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.091, "count": 161, "max_ms": 89.234, "p95_ms": 82.218}}, "truncated_count": 0}, "processing_total": {"avg_ms": 175.967, "count": 161, "max_ms": 237.274, "p95_ms": 218.328}, "read_interval": {"avg_ms": 186.452, "count": 160, "max_ms": 248.203, "p95_ms": 228.094}, "triton_wait": {"avg_ms": 67.305, "count": 161, "max_ms": 92.274, "p95_ms": 85.37}, "visualize": {"avg_ms": 6.475, "count": 161, "max_ms": 11.345, "p95_ms": 8.4}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.839, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11249}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10605.855, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 397, "h264_decode_error_count": 1279, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:00:27+09:00", "last_error_summary": "[h264 @ 0x55ba42a17d40] error while decoding MB 54 16, bytestream -7", "last_exit_at": "2026-07-01T06:00:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.323, "last_started_at": "2026-07-01T05:58:26+09:00", "pid": 82210, "process": {"exists": false, "pid": 82210}, "restart_count": 431, "restart_count_per_hour": 16, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 432}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.423, "sample_count": 1647, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:00:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T06:00:28+09:00", "last_frame_read_age_sec": 0.186, "last_frame_read_at": "2026-07-01T06:00:27+09:00", "mjpeg": {"age_sec": 0.178, "exists": true, "mtime": "2026-07-01T06:00:27.918788+09:00", "mtime_epoch": 1782853227.9187884, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166469}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:00:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5009.086, "name": "celery", "pid": 267, "rss_mb": 5009.086, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.395, "pushed": 5.395, "read": 5.361}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.324, "count": 161, "max_ms": 32.524, "p95_ms": 28.601}, "gstreamer_push": {"avg_ms": 0.939, "count": 161, "max_ms": 1.122, "p95_ms": 1.071}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.596, "count": 161, "max_ms": 45.82, "p95_ms": 34.535}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.278, "count": 161, "max_ms": 92.262, "p95_ms": 85.357}, "yolo11_640_fire_11class": {"avg_ms": 46.343, "count": 161, "max_ms": 64.64, "p95_ms": 61.446}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.091, "count": 161, "max_ms": 89.234, "p95_ms": 82.218}}, "truncated_count": 0}, "processing_total": {"avg_ms": 175.967, "count": 161, "max_ms": 237.274, "p95_ms": 218.328}, "read_interval": {"avg_ms": 186.452, "count": 160, "max_ms": 248.203, "p95_ms": 228.094}, "triton_wait": {"avg_ms": 67.305, "count": 161, "max_ms": 92.274, "p95_ms": 85.37}, "visualize": {"avg_ms": 6.475, "count": 161, "max_ms": 11.345, "p95_ms": 8.4}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.843, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11249}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10714.16, "limit_mb": 8796093022207.996}, "detail": {"pid": 82317, "restart_count": 432, "start_count": 433}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 397, "h264_decode_error_count": 1279, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:00:27+09:00", "last_error_summary": "[h264 @ 0x55ba42a17d40] error while decoding MB 54 16, bytestream -7", "last_exit_at": "2026-07-01T06:00:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:00:30+09:00", "pid": 82317, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.891, "name": "ffmpeg", "pid": 82317, "rss_mb": 138.891, "state": "S (sleeping)", "threads": 106}, "restart_count": 432, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 433}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.044, "sample_count": 1617, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1349903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:00:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.713, "last_frame_pushed_at": "2026-07-01T06:00:28+09:00", "last_frame_read_age_sec": 4.892, "last_frame_read_at": "2026-07-01T06:00:27+09:00", "mjpeg": {"age_sec": 4.695, "exists": true, "mtime": "2026-07-01T06:00:28.106792+09:00", "mtime_epoch": 1782853228.1067924, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164566}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:00:32+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5009.086, "name": "celery", "pid": 267, "rss_mb": 5009.086, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.475, "pushed": 4.475, "read": 4.442}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.353, "count": 133, "max_ms": 32.524, "p95_ms": 28.629}, "gstreamer_push": {"avg_ms": 0.929, "count": 134, "max_ms": 1.122, "p95_ms": 1.078}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.266, "count": 133, "max_ms": 45.82, "p95_ms": 35.764}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.154, "count": 133, "max_ms": 92.262, "p95_ms": 85.477}, "yolo11_640_fire_11class": {"avg_ms": 46.177, "count": 133, "max_ms": 64.64, "p95_ms": 62.243}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.17, "count": 133, "max_ms": 89.234, "p95_ms": 82.528}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.247, "count": 134, "max_ms": 237.274, "p95_ms": 219.363}, "read_interval": {"avg_ms": 189.65, "count": 133, "max_ms": 248.203, "p95_ms": 229.036}, "triton_wait": {"avg_ms": 67.184, "count": 133, "max_ms": 92.274, "p95_ms": 85.49}, "visualize": {"avg_ms": 6.644, "count": 133, "max_ms": 11.345, "p95_ms": 8.504}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11253}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10607.68, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f73ba5c1c0] error while decoding MB 38 10, bytestream -16", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 398, "h264_decode_error_count": 1280, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:02:38+09:00", "last_error_summary": "[h264 @ 0x55f73ba5c1c0] error while decoding MB 38 10, bytestream -16", "last_exit_at": "2026-07-01T06:00:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.245, "last_started_at": "2026-07-01T06:00:30+09:00", "pid": 82317, "process": {"exists": false, "pid": 82317}, "restart_count": 432, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 433}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.561, "sample_count": 1451, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1350474, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:02:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T06:02:37+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T06:02:37+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T06:02:38.001507+09:00", "mtime_epoch": 1782853358.0015073, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168041}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:02:38+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5009.344, "name": "celery", "pid": 267, "rss_mb": 5009.344, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.568, "pushed": 4.568, "read": 4.568}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.111, "count": 136, "max_ms": 36.376, "p95_ms": 32.406}, "gstreamer_push": {"avg_ms": 0.877, "count": 137, "max_ms": 1.321, "p95_ms": 0.952}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.43, "count": 136, "max_ms": 43.053, "p95_ms": 35.888}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.056, "count": 136, "max_ms": 93.973, "p95_ms": 90.073}, "yolo11_640_fire_11class": {"avg_ms": 48.981, "count": 136, "max_ms": 65.512, "p95_ms": 63.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.859, "count": 136, "max_ms": 91.176, "p95_ms": 87.535}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.302, "count": 137, "max_ms": 255.058, "p95_ms": 243.92}, "read_interval": {"avg_ms": 220.162, "count": 137, "max_ms": 265.701, "p95_ms": 253.616}, "triton_wait": {"avg_ms": 73.123, "count": 136, "max_ms": 93.986, "p95_ms": 90.722}, "visualize": {"avg_ms": 7.75, "count": 136, "max_ms": 12.708, "p95_ms": 9.031}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11253}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10606.895, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:02:38+09:00", "last_error_summary": "[h264 @ 0x55f73ba5c1c0] error while decoding MB 38 10, bytestream -16", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 398, "h264_decode_error_count": 1280, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:02:38+09:00", "last_error_summary": "[h264 @ 0x55f73ba5c1c0] error while decoding MB 38 10, bytestream -16", "last_exit_at": "2026-07-01T06:02:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.453, "last_started_at": "2026-07-01T06:00:30+09:00", "pid": 82317, "process": {"exists": false, "pid": 82317}, "restart_count": 432, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 433}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.62, "sample_count": 1451, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1350475, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:02:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:02:38+09:00", "last_frame_read_age_sec": 0.258, "last_frame_read_at": "2026-07-01T06:02:37+09:00", "mjpeg": {"age_sec": 0.253, "exists": true, "mtime": "2026-07-01T06:02:38.001507+09:00", "mtime_epoch": 1782853358.0015073, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168041}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:02:38+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5009.344, "name": "celery", "pid": 267, "rss_mb": 5009.344, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.57, "pushed": 4.57, "read": 4.536}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.088, "count": 136, "max_ms": 36.376, "p95_ms": 32.406}, "gstreamer_push": {"avg_ms": 0.878, "count": 137, "max_ms": 1.321, "p95_ms": 0.952}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.43, "count": 136, "max_ms": 43.053, "p95_ms": 35.888}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.249, "count": 136, "max_ms": 97.57, "p95_ms": 90.711}, "yolo11_640_fire_11class": {"avg_ms": 49.122, "count": 136, "max_ms": 66.336, "p95_ms": 63.343}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.016, "count": 136, "max_ms": 91.176, "p95_ms": 88.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.512, "count": 137, "max_ms": 255.058, "p95_ms": 244.546}, "read_interval": {"avg_ms": 220.05, "count": 136, "max_ms": 265.701, "p95_ms": 253.616}, "triton_wait": {"avg_ms": 73.315, "count": 136, "max_ms": 97.583, "p95_ms": 91.188}, "visualize": {"avg_ms": 7.761, "count": 136, "max_ms": 12.708, "p95_ms": 9.031}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11253}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10606.887, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 398, "h264_decode_error_count": 1280, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:02:38+09:00", "last_error_summary": "[h264 @ 0x55f73ba5c1c0] error while decoding MB 38 10, bytestream -16", "last_exit_at": "2026-07-01T06:02:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.457, "last_started_at": "2026-07-01T06:00:30+09:00", "pid": 82317, "process": {"exists": false, "pid": 82317}, "restart_count": 432, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 433}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.62, "sample_count": 1451, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1350475, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:02:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T06:02:38+09:00", "last_frame_read_age_sec": 0.262, "last_frame_read_at": "2026-07-01T06:02:37+09:00", "mjpeg": {"age_sec": 0.256, "exists": true, "mtime": "2026-07-01T06:02:38.001507+09:00", "mtime_epoch": 1782853358.0015073, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168041}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:02:38+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5009.344, "name": "celery", "pid": 267, "rss_mb": 5009.344, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.569, "pushed": 4.569, "read": 4.536}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.088, "count": 136, "max_ms": 36.376, "p95_ms": 32.406}, "gstreamer_push": {"avg_ms": 0.878, "count": 137, "max_ms": 1.321, "p95_ms": 0.952}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.43, "count": 136, "max_ms": 43.053, "p95_ms": 35.888}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.249, "count": 136, "max_ms": 97.57, "p95_ms": 90.711}, "yolo11_640_fire_11class": {"avg_ms": 49.122, "count": 136, "max_ms": 66.336, "p95_ms": 63.343}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.016, "count": 136, "max_ms": 91.176, "p95_ms": 88.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.512, "count": 137, "max_ms": 255.058, "p95_ms": 244.546}, "read_interval": {"avg_ms": 220.05, "count": 136, "max_ms": 265.701, "p95_ms": 253.616}, "triton_wait": {"avg_ms": 73.315, "count": 136, "max_ms": 97.583, "p95_ms": 91.188}, "visualize": {"avg_ms": 7.761, "count": 136, "max_ms": 12.708, "p95_ms": 9.031}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11253}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10715.688, "limit_mb": 8796093022207.996}, "detail": {"pid": 82424, "restart_count": 433, "start_count": 434}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 398, "h264_decode_error_count": 1280, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:02:38+09:00", "last_error_summary": "[h264 @ 0x55f73ba5c1c0] error while decoding MB 38 10, bytestream -16", "last_exit_at": "2026-07-01T06:02:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:02:40+09:00", "pid": 82424, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.805, "name": "ffmpeg", "pid": 82424, "rss_mb": 138.805, "state": "S (sleeping)", "threads": 106}, "restart_count": 433, "restart_count_per_hour": 18, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 434}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.674, "sample_count": 1425, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1350475, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:02:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.29, "last_frame_pushed_at": "2026-07-01T06:02:38+09:00", "last_frame_read_age_sec": 4.545, "last_frame_read_at": "2026-07-01T06:02:37+09:00", "mjpeg": {"age_sec": 4.276, "exists": true, "mtime": "2026-07-01T06:02:38.265513+09:00", "mtime_epoch": 1782853358.2655127, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162283}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:02:42+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5009.344, "name": "celery", "pid": 267, "rss_mb": 5009.344, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.908, "pushed": 3.908, "read": 3.875}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.147, "count": 116, "max_ms": 36.376, "p95_ms": 33.166}, "gstreamer_push": {"avg_ms": 0.879, "count": 117, "max_ms": 1.321, "p95_ms": 0.952}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.586, "count": 116, "max_ms": 43.053, "p95_ms": 37.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.508, "count": 116, "max_ms": 97.57, "p95_ms": 90.073}, "yolo11_640_fire_11class": {"avg_ms": 48.586, "count": 116, "max_ms": 66.336, "p95_ms": 63.343}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.779, "count": 116, "max_ms": 91.176, "p95_ms": 86.94}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.203, "count": 117, "max_ms": 255.058, "p95_ms": 244.9}, "read_interval": {"avg_ms": 220.695, "count": 116, "max_ms": 265.701, "p95_ms": 254.636}, "triton_wait": {"avg_ms": 72.548, "count": 116, "max_ms": 97.583, "p95_ms": 90.722}, "visualize": {"avg_ms": 7.828, "count": 116, "max_ms": 12.708, "p95_ms": 9.112}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11257}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10608.82, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555784fd6740] error while decoding MB 71 28, bytestream -16", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 399, "h264_decode_error_count": 1281, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:04:40+09:00", "last_error_summary": "[h264 @ 0x555784fd6740] error while decoding MB 71 28, bytestream -16", "last_exit_at": "2026-07-01T06:02:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.219, "last_started_at": "2026-07-01T06:02:40+09:00", "pid": 82424, "process": {"exists": false, "pid": 82424}, "restart_count": 433, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 434}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 302.503, "sample_count": 1254, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1350899, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:04:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T06:04:40+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T06:04:40+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T06:04:40.720071+09:00", "mtime_epoch": 1782853480.7200708, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170435}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:04:40+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5009.602, "name": "celery", "pid": 267, "rss_mb": 5009.602, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.534, "pushed": 3.534, "read": 3.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.312, "count": 105, "max_ms": 34.949, "p95_ms": 32.824}, "gstreamer_push": {"avg_ms": 0.865, "count": 106, "max_ms": 1.333, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.401, "count": 105, "max_ms": 41.674, "p95_ms": 34.281}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.43, "count": 105, "max_ms": 98.667, "p95_ms": 96.373}, "yolo11_640_fire_11class": {"avg_ms": 46.651, "count": 105, "max_ms": 73.657, "p95_ms": 61.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.313, "count": 105, "max_ms": 92.958, "p95_ms": 86.489}}, "truncated_count": 0}, "processing_total": {"avg_ms": 275.715, "count": 106, "max_ms": 350.57, "p95_ms": 324.727}, "read_interval": {"avg_ms": 285.275, "count": 106, "max_ms": 361.084, "p95_ms": 333.853}, "triton_wait": {"avg_ms": 75.508, "count": 105, "max_ms": 98.679, "p95_ms": 96.385}, "visualize": {"avg_ms": 10.268, "count": 105, "max_ms": 13.727, "p95_ms": 12.745}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11257}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10608.082, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:04:40+09:00", "last_error_summary": "[h264 @ 0x555784fd6740] error while decoding MB 71 28, bytestream -16", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 399, "h264_decode_error_count": 1281, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:04:40+09:00", "last_error_summary": "[h264 @ 0x555784fd6740] error while decoding MB 71 28, bytestream -16", "last_exit_at": "2026-07-01T06:04:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.492, "last_started_at": "2026-07-01T06:02:40+09:00", "pid": 82424, "process": {"exists": false, "pid": 82424}, "restart_count": 433, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 434}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 302.904, "sample_count": 1254, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1350900, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:04:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:04:41+09:00", "last_frame_read_age_sec": 0.322, "last_frame_read_at": "2026-07-01T06:04:40+09:00", "mjpeg": {"age_sec": 0.313, "exists": true, "mtime": "2026-07-01T06:04:40.720071+09:00", "mtime_epoch": 1782853480.7200708, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170435}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:04:41+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5009.602, "name": "celery", "pid": 267, "rss_mb": 5009.602, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.534, "pushed": 3.534, "read": 3.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.342, "count": 105, "max_ms": 34.949, "p95_ms": 32.824}, "gstreamer_push": {"avg_ms": 0.864, "count": 106, "max_ms": 1.333, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.406, "count": 105, "max_ms": 41.674, "p95_ms": 34.281}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.435, "count": 105, "max_ms": 98.667, "p95_ms": 96.373}, "yolo11_640_fire_11class": {"avg_ms": 46.881, "count": 105, "max_ms": 73.657, "p95_ms": 63.575}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.55, "count": 105, "max_ms": 92.958, "p95_ms": 86.842}}, "truncated_count": 0}, "processing_total": {"avg_ms": 275.967, "count": 106, "max_ms": 350.57, "p95_ms": 324.727}, "read_interval": {"avg_ms": 285.125, "count": 105, "max_ms": 361.084, "p95_ms": 333.853}, "triton_wait": {"avg_ms": 75.652, "count": 105, "max_ms": 98.679, "p95_ms": 96.385}, "visualize": {"avg_ms": 10.288, "count": 105, "max_ms": 13.727, "p95_ms": 12.745}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11257}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10608.129, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 399, "h264_decode_error_count": 1281, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:04:40+09:00", "last_error_summary": "[h264 @ 0x555784fd6740] error while decoding MB 71 28, bytestream -16", "last_exit_at": "2026-07-01T06:04:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.496, "last_started_at": "2026-07-01T06:02:40+09:00", "pid": 82424, "process": {"exists": false, "pid": 82424}, "restart_count": 433, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 434}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 302.904, "sample_count": 1254, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1350900, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:04:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T06:04:41+09:00", "last_frame_read_age_sec": 0.326, "last_frame_read_at": "2026-07-01T06:04:40+09:00", "mjpeg": {"age_sec": 0.316, "exists": true, "mtime": "2026-07-01T06:04:40.720071+09:00", "mtime_epoch": 1782853480.7200708, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170435}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:04:41+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5009.602, "name": "celery", "pid": 267, "rss_mb": 5009.602, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.534, "pushed": 3.534, "read": 3.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.342, "count": 105, "max_ms": 34.949, "p95_ms": 32.824}, "gstreamer_push": {"avg_ms": 0.864, "count": 106, "max_ms": 1.333, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.406, "count": 105, "max_ms": 41.674, "p95_ms": 34.281}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.435, "count": 105, "max_ms": 98.667, "p95_ms": 96.373}, "yolo11_640_fire_11class": {"avg_ms": 46.881, "count": 105, "max_ms": 73.657, "p95_ms": 63.575}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.55, "count": 105, "max_ms": 92.958, "p95_ms": 86.842}}, "truncated_count": 0}, "processing_total": {"avg_ms": 275.967, "count": 106, "max_ms": 350.57, "p95_ms": 324.727}, "read_interval": {"avg_ms": 285.125, "count": 105, "max_ms": 361.084, "p95_ms": 333.853}, "triton_wait": {"avg_ms": 75.652, "count": 105, "max_ms": 98.679, "p95_ms": 96.385}, "visualize": {"avg_ms": 10.288, "count": 105, "max_ms": 13.727, "p95_ms": 12.745}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11257}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10715.961, "limit_mb": 8796093022207.996}, "detail": {"pid": 82531, "restart_count": 434, "start_count": 435}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 399, "h264_decode_error_count": 1281, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:04:40+09:00", "last_error_summary": "[h264 @ 0x555784fd6740] error while decoding MB 71 28, bytestream -16", "last_exit_at": "2026-07-01T06:04:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:04:43+09:00", "pid": 82531, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.043, "name": "ffmpeg", "pid": 82531, "rss_mb": 139.043, "state": "S (sleeping)", "threads": 106}, "restart_count": 434, "restart_count_per_hour": 18, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 435}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 303.238, "sample_count": 1229, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1350900, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:04:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.514, "last_frame_pushed_at": "2026-07-01T06:04:41+09:00", "last_frame_read_age_sec": 4.834, "last_frame_read_at": "2026-07-01T06:04:40+09:00", "mjpeg": {"age_sec": 4.496, "exists": true, "mtime": "2026-07-01T06:04:41.048078+09:00", "mtime_epoch": 1782853481.0480776, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168015}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:04:45+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5009.602, "name": "celery", "pid": 267, "rss_mb": 5009.602, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 88, "pushed_frames": 88, "read_frames": 87, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.959, "pushed": 2.959, "read": 2.925}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.478, "count": 88, "max_ms": 34.949, "p95_ms": 32.824}, "gstreamer_push": {"avg_ms": 0.867, "count": 88, "max_ms": 1.333, "p95_ms": 0.937}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.224, "count": 88, "max_ms": 41.674, "p95_ms": 33.889}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.83, "count": 88, "max_ms": 98.667, "p95_ms": 96.373}, "yolo11_640_fire_11class": {"avg_ms": 47.185, "count": 88, "max_ms": 73.657, "p95_ms": 63.899}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.126, "count": 88, "max_ms": 92.958, "p95_ms": 86.842}}, "truncated_count": 0}, "processing_total": {"avg_ms": 280.152, "count": 88, "max_ms": 350.57, "p95_ms": 325.127}, "read_interval": {"avg_ms": 289.266, "count": 87, "max_ms": 361.084, "p95_ms": 334.271}, "triton_wait": {"avg_ms": 76.087, "count": 88, "max_ms": 98.679, "p95_ms": 96.385}, "visualize": {"avg_ms": 10.48, "count": 88, "max_ms": 13.727, "p95_ms": 12.777}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.745, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11261}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10609.531, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56073e9144c0] error while decoding MB 6 41, bytestream -46", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 400, "h264_decode_error_count": 1282, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:06:25+09:00", "last_error_summary": "[h264 @ 0x56073e9144c0] error while decoding MB 6 41, bytestream -46", "last_exit_at": "2026-07-01T06:04:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 102.212, "last_started_at": "2026-07-01T06:04:43+09:00", "pid": 82531, "process": {"exists": false, "pid": 82531}, "restart_count": 434, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 435}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 303.238, "sample_count": 1223, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1351373, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:06:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.047, "last_frame_pushed_at": "2026-07-01T06:06:25+09:00", "last_frame_read_age_sec": 0.039, "last_frame_read_at": "2026-07-01T06:06:25+09:00", "mjpeg": {"age_sec": 0.029, "exists": true, "mtime": "2026-07-01T06:06:25.726264+09:00", "mtime_epoch": 1782853585.7262635, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:06:25+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5009.859, "name": "celery", "pid": 267, "rss_mb": 5009.859, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.358, "pushed": 5.358, "read": 5.358}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.356, "count": 160, "max_ms": 33.149, "p95_ms": 29.421}, "gstreamer_push": {"avg_ms": 0.952, "count": 160, "max_ms": 1.155, "p95_ms": 1.09}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.636, "count": 160, "max_ms": 41.408, "p95_ms": 36.152}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.004, "count": 160, "max_ms": 101.069, "p95_ms": 88.125}, "yolo11_640_fire_11class": {"avg_ms": 46.059, "count": 160, "max_ms": 66.069, "p95_ms": 57.845}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.336, "count": 160, "max_ms": 85.835, "p95_ms": 78.122}}, "truncated_count": 0}, "processing_total": {"avg_ms": 177.131, "count": 160, "max_ms": 228.354, "p95_ms": 203.89}, "read_interval": {"avg_ms": 187.375, "count": 160, "max_ms": 237.783, "p95_ms": 214.023}, "triton_wait": {"avg_ms": 75.017, "count": 160, "max_ms": 101.081, "p95_ms": 88.138}, "visualize": {"avg_ms": 6.148, "count": 160, "max_ms": 8.159, "p95_ms": 7.487}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.862, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11261}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10608.84, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:06:25+09:00", "last_error_summary": "[h264 @ 0x56073e9144c0] error while decoding MB 6 41, bytestream -46", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 400, "h264_decode_error_count": 1282, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:06:25+09:00", "last_error_summary": "[h264 @ 0x56073e9144c0] error while decoding MB 6 41, bytestream -46", "last_exit_at": "2026-07-01T06:06:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 102.346, "last_started_at": "2026-07-01T06:04:43+09:00", "pid": 82531, "process": {"exists": false, "pid": 82531}, "restart_count": 434, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 435}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 303.238, "sample_count": 1224, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1351374, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:06:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T06:06:25+09:00", "last_frame_read_age_sec": 0.172, "last_frame_read_at": "2026-07-01T06:06:25+09:00", "mjpeg": {"age_sec": 0.163, "exists": true, "mtime": "2026-07-01T06:06:25.726264+09:00", "mtime_epoch": 1782853585.7262635, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:06:25+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5009.859, "name": "celery", "pid": 267, "rss_mb": 5009.859, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.367, "pushed": 5.367, "read": 5.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.377, "count": 160, "max_ms": 33.149, "p95_ms": 29.421}, "gstreamer_push": {"avg_ms": 0.953, "count": 161, "max_ms": 1.155, "p95_ms": 1.09}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.641, "count": 160, "max_ms": 41.408, "p95_ms": 36.152}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.047, "count": 160, "max_ms": 101.069, "p95_ms": 88.125}, "yolo11_640_fire_11class": {"avg_ms": 46.099, "count": 160, "max_ms": 66.069, "p95_ms": 57.845}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.394, "count": 160, "max_ms": 85.835, "p95_ms": 78.122}}, "truncated_count": 0}, "processing_total": {"avg_ms": 177.062, "count": 161, "max_ms": 228.354, "p95_ms": 203.89}, "read_interval": {"avg_ms": 187.375, "count": 160, "max_ms": 237.783, "p95_ms": 214.023}, "triton_wait": {"avg_ms": 75.061, "count": 160, "max_ms": 101.081, "p95_ms": 88.138}, "visualize": {"avg_ms": 6.15, "count": 160, "max_ms": 8.159, "p95_ms": 7.487}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11261}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10608.832, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 400, "h264_decode_error_count": 1282, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:06:25+09:00", "last_error_summary": "[h264 @ 0x56073e9144c0] error while decoding MB 6 41, bytestream -46", "last_exit_at": "2026-07-01T06:06:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 102.35, "last_started_at": "2026-07-01T06:04:43+09:00", "pid": 82531, "process": {"exists": false, "pid": 82531}, "restart_count": 434, "restart_count_per_hour": 17, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 435}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 303.238, "sample_count": 1224, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1351374, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:06:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T06:06:25+09:00", "last_frame_read_age_sec": 0.176, "last_frame_read_at": "2026-07-01T06:06:25+09:00", "mjpeg": {"age_sec": 0.167, "exists": true, "mtime": "2026-07-01T06:06:25.726264+09:00", "mtime_epoch": 1782853585.7262635, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:06:25+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5009.859, "name": "celery", "pid": 267, "rss_mb": 5009.859, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.367, "pushed": 5.367, "read": 5.333}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.377, "count": 160, "max_ms": 33.149, "p95_ms": 29.421}, "gstreamer_push": {"avg_ms": 0.953, "count": 161, "max_ms": 1.155, "p95_ms": 1.09}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.641, "count": 160, "max_ms": 41.408, "p95_ms": 36.152}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.047, "count": 160, "max_ms": 101.069, "p95_ms": 88.125}, "yolo11_640_fire_11class": {"avg_ms": 46.099, "count": 160, "max_ms": 66.069, "p95_ms": 57.845}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.394, "count": 160, "max_ms": 85.835, "p95_ms": 78.122}}, "truncated_count": 0}, "processing_total": {"avg_ms": 177.062, "count": 161, "max_ms": 228.354, "p95_ms": 203.89}, "read_interval": {"avg_ms": 187.375, "count": 160, "max_ms": 237.783, "p95_ms": 214.023}, "triton_wait": {"avg_ms": 75.061, "count": 160, "max_ms": 101.081, "p95_ms": 88.138}, "visualize": {"avg_ms": 6.15, "count": 160, "max_ms": 8.159, "p95_ms": 7.487}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 418, "tracking_summary": 11261}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10716.859, "limit_mb": 8796093022207.996}, "detail": {"pid": 82638, "restart_count": 435, "start_count": 436}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 400, "h264_decode_error_count": 1282, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:06:25+09:00", "last_error_summary": "[h264 @ 0x56073e9144c0] error while decoding MB 6 41, bytestream -46", "last_exit_at": "2026-07-01T06:06:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:06:28+09:00", "pid": 82638, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.184, "name": "ffmpeg", "pid": 82638, "rss_mb": 139.184, "state": "S (sleeping)", "threads": 106}, "restart_count": 435, "restart_count_per_hour": 18, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 436}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 303.541, "sample_count": 1203, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1351374, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:06:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.535, "last_frame_pushed_at": "2026-07-01T06:06:25+09:00", "last_frame_read_age_sec": 4.702, "last_frame_read_at": "2026-07-01T06:06:25+09:00", "mjpeg": {"age_sec": 4.517, "exists": true, "mtime": "2026-07-01T06:06:25.902267+09:00", "mtime_epoch": 1782853585.9022672, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160715}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:06:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5009.859, "name": "celery", "pid": 267, "rss_mb": 5009.859, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.516, "pushed": 4.516, "read": 4.483}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.387, "count": 135, "max_ms": 33.149, "p95_ms": 29.421}, "gstreamer_push": {"avg_ms": 0.946, "count": 135, "max_ms": 1.155, "p95_ms": 1.095}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.653, "count": 135, "max_ms": 41.408, "p95_ms": 36.152}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.507, "count": 135, "max_ms": 101.069, "p95_ms": 90.814}, "yolo11_640_fire_11class": {"avg_ms": 46.148, "count": 135, "max_ms": 66.069, "p95_ms": 59.701}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.918, "count": 135, "max_ms": 85.835, "p95_ms": 80.314}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.094, "count": 135, "max_ms": 228.354, "p95_ms": 208.347}, "read_interval": {"avg_ms": 189.386, "count": 134, "max_ms": 237.783, "p95_ms": 217.86}, "triton_wait": {"avg_ms": 75.52, "count": 135, "max_ms": 101.081, "p95_ms": 90.827}, "visualize": {"avg_ms": 6.222, "count": 135, "max_ms": 8.159, "p95_ms": 7.531}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 419, "tracking_summary": 11265}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10614.227, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:06:25+09:00", "last_error_summary": "[h264 @ 0x56073e9144c0] error while decoding MB 6 41, bytestream -46", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 400, "h264_decode_error_count": 1282, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:06:25+09:00", "last_error_summary": "[h264 @ 0x56073e9144c0] error while decoding MB 6 41, bytestream -46", "last_exit_at": "2026-07-01T06:08:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.726, "last_started_at": "2026-07-01T06:06:28+09:00", "pid": 82638, "process": {"exists": false, "pid": 82638}, "restart_count": 435, "restart_count_per_hour": 18, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 436}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 302.311, "sample_count": 1254, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1351918, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:08:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:08:34+09:00", "last_frame_read_age_sec": 0.258, "last_frame_read_at": "2026-07-01T06:08:33+09:00", "mjpeg": {"age_sec": 0.251, "exists": true, "mtime": "2026-07-01T06:08:33.892939+09:00", "mtime_epoch": 1782853713.892939, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167447}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:08:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5010.117, "name": "celery", "pid": 267, "rss_mb": 5010.117, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.62, "pushed": 4.62, "read": 4.587}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.764, "count": 138, "max_ms": 32.484, "p95_ms": 31.052}, "gstreamer_push": {"avg_ms": 0.886, "count": 138, "max_ms": 1.089, "p95_ms": 1.008}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.535, "count": 138, "max_ms": 40.121, "p95_ms": 34.632}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.58, "count": 138, "max_ms": 103.139, "p95_ms": 96.043}, "yolo11_640_fire_11class": {"avg_ms": 48.121, "count": 138, "max_ms": 76.327, "p95_ms": 64.181}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.14, "count": 138, "max_ms": 90.763, "p95_ms": 86.453}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.154, "count": 138, "max_ms": 265.216, "p95_ms": 250.301}, "read_interval": {"avg_ms": 217.789, "count": 137, "max_ms": 276.001, "p95_ms": 258.798}, "triton_wait": {"avg_ms": 77.593, "count": 138, "max_ms": 103.151, "p95_ms": 96.057}, "visualize": {"avg_ms": 7.31, "count": 138, "max_ms": 9.458, "p95_ms": 9.187}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 419, "tracking_summary": 11265}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10614.219, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 400, "h264_decode_error_count": 1282, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:06:25+09:00", "last_error_summary": "[h264 @ 0x56073e9144c0] error while decoding MB 6 41, bytestream -46", "last_exit_at": "2026-07-01T06:08:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.729, "last_started_at": "2026-07-01T06:06:28+09:00", "pid": 82638, "process": {"exists": false, "pid": 82638}, "restart_count": 435, "restart_count_per_hour": 18, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 436}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 302.311, "sample_count": 1254, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1351918, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:08:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T06:08:34+09:00", "last_frame_read_age_sec": 0.261, "last_frame_read_at": "2026-07-01T06:08:33+09:00", "mjpeg": {"age_sec": 0.255, "exists": true, "mtime": "2026-07-01T06:08:33.892939+09:00", "mtime_epoch": 1782853713.892939, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167447}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:08:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5010.117, "name": "celery", "pid": 267, "rss_mb": 5010.117, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.62, "pushed": 4.62, "read": 4.586}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.764, "count": 138, "max_ms": 32.484, "p95_ms": 31.052}, "gstreamer_push": {"avg_ms": 0.886, "count": 138, "max_ms": 1.089, "p95_ms": 1.008}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.535, "count": 138, "max_ms": 40.121, "p95_ms": 34.632}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.58, "count": 138, "max_ms": 103.139, "p95_ms": 96.043}, "yolo11_640_fire_11class": {"avg_ms": 48.121, "count": 138, "max_ms": 76.327, "p95_ms": 64.181}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.14, "count": 138, "max_ms": 90.763, "p95_ms": 86.453}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.154, "count": 138, "max_ms": 265.216, "p95_ms": 250.301}, "read_interval": {"avg_ms": 217.789, "count": 137, "max_ms": 276.001, "p95_ms": 258.798}, "triton_wait": {"avg_ms": 77.593, "count": 138, "max_ms": 103.151, "p95_ms": 96.057}, "visualize": {"avg_ms": 7.31, "count": 138, "max_ms": 9.458, "p95_ms": 9.187}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.872, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 419, "tracking_summary": 11265}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10720.363, "limit_mb": 8796093022207.996}, "detail": {"pid": 82805, "restart_count": 436, "start_count": 437}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 400, "h264_decode_error_count": 1282, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:06:25+09:00", "last_error_summary": "[h264 @ 0x56073e9144c0] error while decoding MB 6 41, bytestream -46", "last_exit_at": "2026-07-01T06:08:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:08:36+09:00", "pid": 82805, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.297, "name": "ffmpeg", "pid": 82805, "rss_mb": 136.297, "state": "S (sleeping)", "threads": 106}, "restart_count": 436, "restart_count_per_hour": 19, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 437}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 299.237, "sample_count": 1240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1351918, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:08:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.386, "last_frame_pushed_at": "2026-07-01T06:08:34+09:00", "last_frame_read_age_sec": 4.641, "last_frame_read_at": "2026-07-01T06:08:33+09:00", "mjpeg": {"age_sec": 4.374, "exists": true, "mtime": "2026-07-01T06:08:34.152945+09:00", "mtime_epoch": 1782853714.1529446, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167812}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:08:38+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5010.117, "name": "celery", "pid": 267, "rss_mb": 5010.117, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.007, "pushed": 4.007, "read": 3.974}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.638, "count": 119, "max_ms": 31.785, "p95_ms": 30.831}, "gstreamer_push": {"avg_ms": 0.889, "count": 120, "max_ms": 1.089, "p95_ms": 1.008}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.425, "count": 119, "max_ms": 38.604, "p95_ms": 34.632}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.21, "count": 119, "max_ms": 103.139, "p95_ms": 96.921}, "yolo11_640_fire_11class": {"avg_ms": 47.604, "count": 119, "max_ms": 76.327, "p95_ms": 64.181}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.027, "count": 119, "max_ms": 90.763, "p95_ms": 84.274}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.972, "count": 120, "max_ms": 256.811, "p95_ms": 246.005}, "read_interval": {"avg_ms": 214.511, "count": 119, "max_ms": 267.289, "p95_ms": 255.426}, "triton_wait": {"avg_ms": 77.223, "count": 119, "max_ms": 103.151, "p95_ms": 96.932}, "visualize": {"avg_ms": 7.176, "count": 119, "max_ms": 9.454, "p95_ms": 8.598}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 419, "tracking_summary": 11270}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10615.867, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559d147df980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d147df980] error while decoding MB 0 25, bytestream 55424 [h264 @ 0x559d146c0000] error while decoding MB 31 13, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 401, "h264_decode_error_count": 1284, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:10:55+09:00", "last_error_summary": "[h264 @ 0x559d147df980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d147df980] error while decoding MB 0 25, bytestream 55424 [h264 @ 0x559d146c0000] error while decoding MB 31 13, bytestream -7", "last_exit_at": "2026-07-01T06:08:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.077, "last_started_at": "2026-07-01T06:08:36+09:00", "pid": 82805, "process": {"exists": false, "pid": 82805}, "restart_count": 436, "restart_count_per_hour": 18, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 437}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 312.985, "sample_count": 1294, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1352506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:10:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T06:10:55+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T06:10:55+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T06:10:55.563896+09:00", "mtime_epoch": 1782853855.563896, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170795}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:10:55+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5010.375, "name": "celery", "pid": 267, "rss_mb": 5010.375, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 91, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.036, "pushed": 3.036, "read": 3.036}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.379, "count": 90, "max_ms": 44.933, "p95_ms": 38.751}, "gstreamer_push": {"avg_ms": 0.849, "count": 91, "max_ms": 0.975, "p95_ms": 0.88}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.236, "count": 90, "max_ms": 42.606, "p95_ms": 36.053}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.045, "count": 90, "max_ms": 106.461, "p95_ms": 96.181}, "yolo11_640_fire_11class": {"avg_ms": 49.269, "count": 90, "max_ms": 72.827, "p95_ms": 68.369}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.7, "count": 90, "max_ms": 97.677, "p95_ms": 91.605}}, "truncated_count": 0}, "processing_total": {"avg_ms": 321.797, "count": 91, "max_ms": 406.828, "p95_ms": 394.399}, "read_interval": {"avg_ms": 331.473, "count": 91, "max_ms": 417.295, "p95_ms": 403.705}, "triton_wait": {"avg_ms": 77.459, "count": 90, "max_ms": 106.473, "p95_ms": 97.69}, "visualize": {"avg_ms": 11.162, "count": 90, "max_ms": 14.499, "p95_ms": 13.237}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 419, "tracking_summary": 11270}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10614.699, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:10:55+09:00", "last_error_summary": "[h264 @ 0x559d147df980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d147df980] error while decoding MB 0 25, bytestream 55424 [h264 @ 0x559d146c0000] error while decoding MB 31 13, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 401, "h264_decode_error_count": 1284, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:10:55+09:00", "last_error_summary": "[h264 @ 0x559d147df980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d147df980] error while decoding MB 0 25, bytestream 55424 [h264 @ 0x559d146c0000] error while decoding MB 31 13, bytestream -7", "last_exit_at": "2026-07-01T06:10:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.388, "last_started_at": "2026-07-01T06:08:36+09:00", "pid": 82805, "process": {"exists": false, "pid": 82805}, "restart_count": 436, "restart_count_per_hour": 18, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 437}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 313.194, "sample_count": 1293, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1352507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:10:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T06:10:55+09:00", "last_frame_read_age_sec": 0.361, "last_frame_read_at": "2026-07-01T06:10:55+09:00", "mjpeg": {"age_sec": 0.35, "exists": true, "mtime": "2026-07-01T06:10:55.563896+09:00", "mtime_epoch": 1782853855.563896, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170795}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:10:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5010.375, "name": "celery", "pid": 267, "rss_mb": 5010.375, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 89, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.02, "pushed": 3.02, "read": 2.987}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.479, "count": 90, "max_ms": 44.933, "p95_ms": 38.751}, "gstreamer_push": {"avg_ms": 0.849, "count": 90, "max_ms": 0.975, "p95_ms": 0.878}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.291, "count": 90, "max_ms": 42.606, "p95_ms": 36.053}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.133, "count": 90, "max_ms": 106.461, "p95_ms": 96.181}, "yolo11_640_fire_11class": {"avg_ms": 49.212, "count": 90, "max_ms": 72.827, "p95_ms": 68.369}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.602, "count": 90, "max_ms": 97.677, "p95_ms": 91.605}}, "truncated_count": 0}, "processing_total": {"avg_ms": 324.056, "count": 90, "max_ms": 406.828, "p95_ms": 394.399}, "read_interval": {"avg_ms": 333.406, "count": 89, "max_ms": 417.295, "p95_ms": 403.705}, "triton_wait": {"avg_ms": 77.547, "count": 90, "max_ms": 106.473, "p95_ms": 97.69}, "visualize": {"avg_ms": 11.178, "count": 90, "max_ms": 14.499, "p95_ms": 13.237}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.797, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 419, "tracking_summary": 11270}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10614.691, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 401, "h264_decode_error_count": 1284, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:10:55+09:00", "last_error_summary": "[h264 @ 0x559d147df980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d147df980] error while decoding MB 0 25, bytestream 55424 [h264 @ 0x559d146c0000] error while decoding MB 31 13, bytestream -7", "last_exit_at": "2026-07-01T06:10:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.391, "last_started_at": "2026-07-01T06:08:36+09:00", "pid": 82805, "process": {"exists": false, "pid": 82805}, "restart_count": 436, "restart_count_per_hour": 18, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 437}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 313.194, "sample_count": 1293, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1352507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:10:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T06:10:55+09:00", "last_frame_read_age_sec": 0.365, "last_frame_read_at": "2026-07-01T06:10:55+09:00", "mjpeg": {"age_sec": 0.353, "exists": true, "mtime": "2026-07-01T06:10:55.563896+09:00", "mtime_epoch": 1782853855.563896, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170795}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:10:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5010.375, "name": "celery", "pid": 267, "rss_mb": 5010.375, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 89, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.02, "pushed": 3.02, "read": 2.987}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.479, "count": 90, "max_ms": 44.933, "p95_ms": 38.751}, "gstreamer_push": {"avg_ms": 0.849, "count": 90, "max_ms": 0.975, "p95_ms": 0.878}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.291, "count": 90, "max_ms": 42.606, "p95_ms": 36.053}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.133, "count": 90, "max_ms": 106.461, "p95_ms": 96.181}, "yolo11_640_fire_11class": {"avg_ms": 49.212, "count": 90, "max_ms": 72.827, "p95_ms": 68.369}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.602, "count": 90, "max_ms": 97.677, "p95_ms": 91.605}}, "truncated_count": 0}, "processing_total": {"avg_ms": 324.056, "count": 90, "max_ms": 406.828, "p95_ms": 394.399}, "read_interval": {"avg_ms": 333.406, "count": 89, "max_ms": 417.295, "p95_ms": 403.705}, "triton_wait": {"avg_ms": 77.547, "count": 90, "max_ms": 106.473, "p95_ms": 97.69}, "visualize": {"avg_ms": 11.178, "count": 90, "max_ms": 14.499, "p95_ms": 13.237}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.8, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 419, "tracking_summary": 11270}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10723.473, "limit_mb": 8796093022207.996}, "detail": {"pid": 82912, "restart_count": 437, "start_count": 438}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 401, "h264_decode_error_count": 1284, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:10:55+09:00", "last_error_summary": "[h264 @ 0x559d147df980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559d147df980] error while decoding MB 0 25, bytestream 55424 [h264 @ 0x559d146c0000] error while decoding MB 31 13, bytestream -7", "last_exit_at": "2026-07-01T06:10:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:10:58+09:00", "pid": 82912, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.836, "name": "ffmpeg", "pid": 82912, "rss_mb": 138.836, "state": "S (sleeping)", "threads": 106}, "restart_count": 437, "restart_count_per_hour": 19, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 438}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 317.142, "sample_count": 1267, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1352507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:10:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.64, "last_frame_pushed_at": "2026-07-01T06:10:55+09:00", "last_frame_read_age_sec": 4.997, "last_frame_read_at": "2026-07-01T06:10:55+09:00", "mjpeg": {"age_sec": 4.621, "exists": true, "mtime": "2026-07-01T06:10:55.927903+09:00", "mtime_epoch": 1782853855.9279034, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171064}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:11:00+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5010.375, "name": "celery", "pid": 267, "rss_mb": 5010.375, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.445, "pushed": 2.445, "read": 2.412}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.483, "count": 72, "max_ms": 44.933, "p95_ms": 38.812}, "gstreamer_push": {"avg_ms": 0.851, "count": 73, "max_ms": 0.975, "p95_ms": 0.88}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.7, "count": 72, "max_ms": 42.606, "p95_ms": 36.053}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.009, "count": 72, "max_ms": 106.461, "p95_ms": 99.12}, "yolo11_640_fire_11class": {"avg_ms": 48.859, "count": 72, "max_ms": 72.827, "p95_ms": 69.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.389, "count": 72, "max_ms": 97.677, "p95_ms": 92.08}}, "truncated_count": 0}, "processing_total": {"avg_ms": 340.234, "count": 73, "max_ms": 406.828, "p95_ms": 399.672}, "read_interval": {"avg_ms": 349.775, "count": 72, "max_ms": 417.295, "p95_ms": 408.954}, "triton_wait": {"avg_ms": 77.523, "count": 72, "max_ms": 106.473, "p95_ms": 99.132}, "visualize": {"avg_ms": 11.642, "count": 72, "max_ms": 14.499, "p95_ms": 13.316}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.852, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 419, "tracking_summary": 11276}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10616.938, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55eda7e73c00] cabac decode of qscale diff failed at 76 33 [h264 @ 0x55eda7e73c00] error while decoding MB 76 33, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 402, "h264_decode_error_count": 1286, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:13:00+09:00", "last_error_summary": "[h264 @ 0x55eda7e73c00] cabac decode of qscale diff failed at 76 33 [h264 @ 0x55eda7e73c00] error while decoding MB 76 33, bytestream -7", "last_exit_at": "2026-07-01T06:10:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.573, "last_started_at": "2026-07-01T06:10:58+09:00", "pid": 82912, "process": {"exists": false, "pid": 82912}, "restart_count": 437, "restart_count_per_hour": 19, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 438}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 289.267, "sample_count": 1389, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1353150, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:12:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T06:13:00+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T06:13:00+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T06:13:00.082494+09:00", "mtime_epoch": 1782853980.0824943, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170908}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:13:00+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5010.633, "name": "celery", "pid": 267, "rss_mb": 5010.633, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.137, "pushed": 6.137, "read": 6.137}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.277, "count": 183, "max_ms": 39.006, "p95_ms": 30.299}, "gstreamer_push": {"avg_ms": 1.051, "count": 184, "max_ms": 1.352, "p95_ms": 1.25}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.589, "count": 183, "max_ms": 39.39, "p95_ms": 36.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.385, "count": 183, "max_ms": 101.551, "p95_ms": 91.313}, "yolo11_640_fire_11class": {"avg_ms": 46.788, "count": 183, "max_ms": 69.207, "p95_ms": 63.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.945, "count": 183, "max_ms": 89.842, "p95_ms": 81.959}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.272, "count": 184, "max_ms": 219.263, "p95_ms": 198.538}, "read_interval": {"avg_ms": 163.636, "count": 184, "max_ms": 228.85, "p95_ms": 208.403}, "triton_wait": {"avg_ms": 70.404, "count": 183, "max_ms": 101.562, "p95_ms": 91.325}, "visualize": {"avg_ms": 5.177, "count": 183, "max_ms": 8.742, "p95_ms": 7.244}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 419, "tracking_summary": 11276}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10616.449, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:13:00+09:00", "last_error_summary": "[h264 @ 0x55eda7e73c00] cabac decode of qscale diff failed at 76 33 [h264 @ 0x55eda7e73c00] error while decoding MB 76 33, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 402, "h264_decode_error_count": 1286, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:13:00+09:00", "last_error_summary": "[h264 @ 0x55eda7e73c00] cabac decode of qscale diff failed at 76 33 [h264 @ 0x55eda7e73c00] error while decoding MB 76 33, bytestream -7", "last_exit_at": "2026-07-01T06:13:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.683, "last_started_at": "2026-07-01T06:10:58+09:00", "pid": 82912, "process": {"exists": false, "pid": 82912}, "restart_count": 437, "restart_count_per_hour": 19, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 438}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 289.267, "sample_count": 1389, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1353151, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:13:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T06:13:00+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-01T06:13:00+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-01T06:13:00.082494+09:00", "mtime_epoch": 1782853980.0824943, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170908}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:13:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5010.633, "name": "celery", "pid": 267, "rss_mb": 5010.633, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.157, "pushed": 6.157, "read": 6.124}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.289, "count": 184, "max_ms": 39.006, "p95_ms": 30.299}, "gstreamer_push": {"avg_ms": 1.052, "count": 184, "max_ms": 1.352, "p95_ms": 1.25}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.58, "count": 184, "max_ms": 39.39, "p95_ms": 36.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.344, "count": 184, "max_ms": 101.551, "p95_ms": 91.313}, "yolo11_640_fire_11class": {"avg_ms": 46.757, "count": 184, "max_ms": 69.207, "p95_ms": 63.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.929, "count": 184, "max_ms": 89.842, "p95_ms": 81.959}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.147, "count": 184, "max_ms": 219.263, "p95_ms": 198.538}, "read_interval": {"avg_ms": 163.503, "count": 183, "max_ms": 228.85, "p95_ms": 208.403}, "triton_wait": {"avg_ms": 70.363, "count": 184, "max_ms": 101.562, "p95_ms": 91.325}, "visualize": {"avg_ms": 5.179, "count": 184, "max_ms": 8.742, "p95_ms": 7.244}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 419, "tracking_summary": 11276}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10616.387, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 402, "h264_decode_error_count": 1286, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:13:00+09:00", "last_error_summary": "[h264 @ 0x55eda7e73c00] cabac decode of qscale diff failed at 76 33 [h264 @ 0x55eda7e73c00] error while decoding MB 76 33, bytestream -7", "last_exit_at": "2026-07-01T06:13:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.687, "last_started_at": "2026-07-01T06:10:58+09:00", "pid": 82912, "process": {"exists": false, "pid": 82912}, "restart_count": 437, "restart_count_per_hour": 19, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 438}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 289.267, "sample_count": 1389, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1353151, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:13:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T06:13:00+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-01T06:13:00+09:00", "mjpeg": {"age_sec": 0.153, "exists": true, "mtime": "2026-07-01T06:13:00.082494+09:00", "mtime_epoch": 1782853980.0824943, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170908}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:13:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5010.633, "name": "celery", "pid": 267, "rss_mb": 5010.633, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.156, "pushed": 6.156, "read": 6.123}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.289, "count": 184, "max_ms": 39.006, "p95_ms": 30.299}, "gstreamer_push": {"avg_ms": 1.052, "count": 184, "max_ms": 1.352, "p95_ms": 1.25}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.58, "count": 184, "max_ms": 39.39, "p95_ms": 36.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.344, "count": 184, "max_ms": 101.551, "p95_ms": 91.313}, "yolo11_640_fire_11class": {"avg_ms": 46.757, "count": 184, "max_ms": 69.207, "p95_ms": 63.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.929, "count": 184, "max_ms": 89.842, "p95_ms": 81.959}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.147, "count": 184, "max_ms": 219.263, "p95_ms": 198.538}, "read_interval": {"avg_ms": 163.503, "count": 183, "max_ms": 228.85, "p95_ms": 208.403}, "triton_wait": {"avg_ms": 70.363, "count": 184, "max_ms": 101.562, "p95_ms": 91.325}, "visualize": {"avg_ms": 5.179, "count": 184, "max_ms": 8.742, "p95_ms": 7.244}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 419, "tracking_summary": 11276}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10724.648, "limit_mb": 8796093022207.996}, "detail": {"pid": 83019, "restart_count": 438, "start_count": 439}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 402, "h264_decode_error_count": 1286, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:13:00+09:00", "last_error_summary": "[h264 @ 0x55eda7e73c00] cabac decode of qscale diff failed at 76 33 [h264 @ 0x55eda7e73c00] error while decoding MB 76 33, bytestream -7", "last_exit_at": "2026-07-01T06:13:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:13:02+09:00", "pid": 83019, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.766, "name": "ffmpeg", "pid": 83019, "rss_mb": 138.766, "state": "S (sleeping)", "threads": 106}, "restart_count": 438, "restart_count_per_hour": 20, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 439}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 289.597, "sample_count": 1370, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1353151, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:13:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.203, "last_frame_pushed_at": "2026-07-01T06:13:00+09:00", "last_frame_read_age_sec": 4.359, "last_frame_read_at": "2026-07-01T06:13:00+09:00", "mjpeg": {"age_sec": 4.181, "exists": true, "mtime": "2026-07-01T06:13:00.250498+09:00", "mtime_epoch": 1782853980.2504978, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169714}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:13:04+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5010.633, "name": "celery", "pid": 267, "rss_mb": 5010.633, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 159, "pushed_frames": 159, "read_frames": 158, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.308, "pushed": 5.308, "read": 5.275}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.314, "count": 158, "max_ms": 39.006, "p95_ms": 30.356}, "gstreamer_push": {"avg_ms": 1.056, "count": 159, "max_ms": 1.352, "p95_ms": 1.256}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.69, "count": 158, "max_ms": 39.39, "p95_ms": 36.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.507, "count": 158, "max_ms": 101.551, "p95_ms": 91.407}, "yolo11_640_fire_11class": {"avg_ms": 46.589, "count": 158, "max_ms": 67.685, "p95_ms": 63.612}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.624, "count": 158, "max_ms": 89.842, "p95_ms": 82.053}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.81, "count": 159, "max_ms": 219.263, "p95_ms": 205.478}, "read_interval": {"avg_ms": 163.229, "count": 158, "max_ms": 228.85, "p95_ms": 215.138}, "triton_wait": {"avg_ms": 70.52, "count": 158, "max_ms": 101.562, "p95_ms": 91.419}, "visualize": {"avg_ms": 5.175, "count": 158, "max_ms": 8.742, "p95_ms": 7.297}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 420, "tracking_summary": 11279}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10774.508, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 403, "h264_decode_error_count": 1287, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "last_exit_at": "2026-07-01T06:13:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.341, "last_started_at": "2026-07-01T06:13:02+09:00", "pid": 83019, "process": {"exists": false, "pid": 83019}, "restart_count": 438, "restart_count_per_hour": 19, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 439}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 332.57, "sample_count": 1263, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1353567, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:14:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T06:14:57+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T06:14:57+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T06:14:57.728949+09:00", "mtime_epoch": 1782854097.7289488, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173073}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:14:57+09:00", "process": {"exists": true, "fd_count": 139, "hwm_mb": 5010.891, "name": "celery", "pid": 267, "rss_mb": 5010.891, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": 2.037, "recorders": [{"age_sec": 2.037, "frame_count": 56, "is_recording": true, "is_stopping": false, "pid": 83126, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.82, "name": "ffmpeg", "pid": 83126, "rss_mb": 185.82, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 44, "returncode": null, "running": true, "started_at": "2026-07-01T06:14:55+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_061455_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [83126], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.615, "pushed": 3.615, "read": 3.615}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.294, "count": 108, "max_ms": 43.397, "p95_ms": 36.971}, "gstreamer_push": {"avg_ms": 0.869, "count": 108, "max_ms": 1.002, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.52, "count": 108, "max_ms": 67.01, "p95_ms": 48.04}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.566, "count": 108, "max_ms": 93.801, "p95_ms": 87.128}, "yolo11_640_fire_11class": {"avg_ms": 47.431, "count": 108, "max_ms": 73.865, "p95_ms": 68.54}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.508, "count": 108, "max_ms": 97.648, "p95_ms": 83.316}}, "truncated_count": 0}, "processing_total": {"avg_ms": 268.301, "count": 108, "max_ms": 920.386, "p95_ms": 324.069}, "read_interval": {"avg_ms": 277.987, "count": 108, "max_ms": 930.927, "p95_ms": 334.6}, "triton_wait": {"avg_ms": 65.822, "count": 108, "max_ms": 97.66, "p95_ms": 88.124}, "visualize": {"avg_ms": 8.063, "count": 108, "max_ms": 13.329, "p95_ms": 10.351}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.872, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 420, "tracking_summary": 11279}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10773.82, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 403, "h264_decode_error_count": 1287, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "last_exit_at": "2026-07-01T06:14:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.524, "last_started_at": "2026-07-01T06:13:02+09:00", "pid": 83019, "process": {"exists": false, "pid": 83019}, "restart_count": 438, "restart_count_per_hour": 19, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 439}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 332.57, "sample_count": 1264, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1353568, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:14:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:14:57+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T06:14:57+09:00", "mjpeg": {"age_sec": 0.225, "exists": true, "mtime": "2026-07-01T06:14:57.728949+09:00", "mtime_epoch": 1782854097.7289488, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173073}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:14:57+09:00", "process": {"exists": true, "fd_count": 138, "hwm_mb": 5010.891, "name": "celery", "pid": 267, "rss_mb": 5010.891, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": 2.216, "recorders": [{"age_sec": 2.216, "frame_count": 57, "is_recording": true, "is_stopping": false, "pid": 83126, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.82, "name": "ffmpeg", "pid": 83126, "rss_mb": 185.82, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 43, "returncode": null, "running": true, "started_at": "2026-07-01T06:14:55+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_061455_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [83126], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.622, "pushed": 3.622, "read": 3.588}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.298, "count": 108, "max_ms": 43.397, "p95_ms": 36.971}, "gstreamer_push": {"avg_ms": 0.869, "count": 108, "max_ms": 1.002, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.525, "count": 108, "max_ms": 67.01, "p95_ms": 48.04}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.725, "count": 108, "max_ms": 93.801, "p95_ms": 87.128}, "yolo11_640_fire_11class": {"avg_ms": 47.621, "count": 108, "max_ms": 73.865, "p95_ms": 68.54}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.685, "count": 108, "max_ms": 97.648, "p95_ms": 83.316}}, "truncated_count": 0}, "processing_total": {"avg_ms": 268.544, "count": 108, "max_ms": 920.386, "p95_ms": 324.069}, "read_interval": {"avg_ms": 278.613, "count": 107, "max_ms": 930.927, "p95_ms": 334.6}, "triton_wait": {"avg_ms": 65.981, "count": 108, "max_ms": 97.66, "p95_ms": 88.124}, "visualize": {"avg_ms": 8.07, "count": 108, "max_ms": 13.329, "p95_ms": 10.351}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.818, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 420, "tracking_summary": 11279}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10773.812, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 403, "h264_decode_error_count": 1287, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "last_exit_at": "2026-07-01T06:14:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.528, "last_started_at": "2026-07-01T06:13:02+09:00", "pid": 83019, "process": {"exists": false, "pid": 83019}, "restart_count": 438, "restart_count_per_hour": 19, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 439}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 332.57, "sample_count": 1264, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1353568, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:14:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T06:14:57+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-01T06:14:57+09:00", "mjpeg": {"age_sec": 0.229, "exists": true, "mtime": "2026-07-01T06:14:57.728949+09:00", "mtime_epoch": 1782854097.7289488, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173073}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:14:57+09:00", "process": {"exists": true, "fd_count": 138, "hwm_mb": 5010.891, "name": "celery", "pid": 267, "rss_mb": 5010.891, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": 2.22, "recorders": [{"age_sec": 2.22, "frame_count": 57, "is_recording": true, "is_stopping": false, "pid": 83126, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.82, "name": "ffmpeg", "pid": 83126, "rss_mb": 185.82, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 43, "returncode": null, "running": true, "started_at": "2026-07-01T06:14:55+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_061455_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [83126], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.621, "pushed": 3.621, "read": 3.588}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.298, "count": 108, "max_ms": 43.397, "p95_ms": 36.971}, "gstreamer_push": {"avg_ms": 0.869, "count": 108, "max_ms": 1.002, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.525, "count": 108, "max_ms": 67.01, "p95_ms": 48.04}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.725, "count": 108, "max_ms": 93.801, "p95_ms": 87.128}, "yolo11_640_fire_11class": {"avg_ms": 47.621, "count": 108, "max_ms": 73.865, "p95_ms": 68.54}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.685, "count": 108, "max_ms": 97.648, "p95_ms": 83.316}}, "truncated_count": 0}, "processing_total": {"avg_ms": 268.544, "count": 108, "max_ms": 920.386, "p95_ms": 324.069}, "read_interval": {"avg_ms": 278.613, "count": 107, "max_ms": 930.927, "p95_ms": 334.6}, "triton_wait": {"avg_ms": 65.981, "count": 108, "max_ms": 97.66, "p95_ms": 88.124}, "visualize": {"avg_ms": 8.07, "count": 108, "max_ms": 13.329, "p95_ms": 10.351}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.822, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 420, "tracking_summary": 11279}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10881.508, "limit_mb": 8796093022207.996}, "detail": {"pid": 83186, "restart_count": 439, "start_count": 440}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 403, "h264_decode_error_count": 1287, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "last_exit_at": "2026-07-01T06:15:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:15:00+09:00", "pid": 83186, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.297, "name": "ffmpeg", "pid": 83186, "rss_mb": 138.297, "state": "S (sleeping)", "threads": 106}, "restart_count": 439, "restart_count_per_hour": 20, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 440}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 333.791, "sample_count": 1247, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1353568, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:14:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.487, "last_frame_pushed_at": "2026-07-01T06:14:57+09:00", "last_frame_read_age_sec": 4.716, "last_frame_read_at": "2026-07-01T06:14:57+09:00", "mjpeg": {"age_sec": 4.473, "exists": true, "mtime": "2026-07-01T06:14:57.964954+09:00", "mtime_epoch": 1782854097.9649537, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172696}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:15:02+09:00", "process": {"exists": true, "fd_count": 139, "hwm_mb": 5010.891, "name": "celery", "pid": 267, "rss_mb": 5010.891, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": 6.701, "recorders": [{"age_sec": 6.701, "frame_count": 57, "is_recording": true, "is_stopping": false, "pid": 83126, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.82, "name": "ffmpeg", "pid": 83126, "rss_mb": 185.82, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 43, "returncode": null, "running": true, "started_at": "2026-07-01T06:14:55+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_061455_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [83126], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 88, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.986, "pushed": 2.986, "read": 2.953}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.788, "count": 89, "max_ms": 43.397, "p95_ms": 37.037}, "gstreamer_push": {"avg_ms": 0.87, "count": 89, "max_ms": 1.002, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.377, "count": 89, "max_ms": 67.01, "p95_ms": 48.473}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.652, "count": 89, "max_ms": 93.574, "p95_ms": 87.128}, "yolo11_640_fire_11class": {"avg_ms": 47.235, "count": 89, "max_ms": 73.865, "p95_ms": 68.423}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.324, "count": 89, "max_ms": 97.648, "p95_ms": 82.333}}, "truncated_count": 0}, "processing_total": {"avg_ms": 278.094, "count": 89, "max_ms": 920.386, "p95_ms": 327.135}, "read_interval": {"avg_ms": 288.365, "count": 88, "max_ms": 930.927, "p95_ms": 337.448}, "triton_wait": {"avg_ms": 65.947, "count": 89, "max_ms": 97.66, "p95_ms": 87.14}, "visualize": {"avg_ms": 8.188, "count": 89, "max_ms": 13.329, "p95_ms": 10.514}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.802, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 422, "tracking_summary": 11282}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10625.508, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 403, "h264_decode_error_count": 1287, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "last_exit_at": "2026-07-01T06:17:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.024, "last_started_at": "2026-07-01T06:15:00+09:00", "pid": 83186, "process": {"exists": false, "pid": 83186}, "restart_count": 439, "restart_count_per_hour": 20, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 440}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 389.017, "sample_count": 1119, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1353939, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:17:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:17:02+09:00", "last_frame_read_age_sec": 0.32, "last_frame_read_at": "2026-07-01T06:17:02+09:00", "mjpeg": {"age_sec": 0.309, "exists": true, "mtime": "2026-07-01T06:17:02.151544+09:00", "mtime_epoch": 1782854222.1515443, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170369}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:17:02+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5011.148, "name": "celery", "pid": 267, "rss_mb": 5011.148, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.165, "pushed": 3.165, "read": 3.131}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.09, "count": 94, "max_ms": 50.379, "p95_ms": 42.084}, "gstreamer_push": {"avg_ms": 0.863, "count": 94, "max_ms": 1.253, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.771, "count": 94, "max_ms": 46.064, "p95_ms": 42.184}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.239, "count": 94, "max_ms": 103.123, "p95_ms": 94.263}, "yolo11_640_fire_11class": {"avg_ms": 49.377, "count": 94, "max_ms": 71.654, "p95_ms": 70.25}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.688, "count": 94, "max_ms": 96.498, "p95_ms": 86.455}}, "truncated_count": 0}, "processing_total": {"avg_ms": 310.408, "count": 94, "max_ms": 474.366, "p95_ms": 386.178}, "read_interval": {"avg_ms": 319.977, "count": 93, "max_ms": 483.303, "p95_ms": 395.341}, "triton_wait": {"avg_ms": 73.352, "count": 94, "max_ms": 103.135, "p95_ms": 94.276}, "visualize": {"avg_ms": 11.889, "count": 94, "max_ms": 20.244, "p95_ms": 18.292}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.7, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 422, "tracking_summary": 11282}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10625.5, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 403, "h264_decode_error_count": 1287, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "last_exit_at": "2026-07-01T06:17:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.028, "last_started_at": "2026-07-01T06:15:00+09:00", "pid": 83186, "process": {"exists": false, "pid": 83186}, "restart_count": 439, "restart_count_per_hour": 20, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 440}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 389.017, "sample_count": 1119, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1353939, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:17:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T06:17:02+09:00", "last_frame_read_age_sec": 0.323, "last_frame_read_at": "2026-07-01T06:17:02+09:00", "mjpeg": {"age_sec": 0.313, "exists": true, "mtime": "2026-07-01T06:17:02.151544+09:00", "mtime_epoch": 1782854222.1515443, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170369}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:17:02+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5011.148, "name": "celery", "pid": 267, "rss_mb": 5011.148, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.165, "pushed": 3.165, "read": 3.131}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.09, "count": 94, "max_ms": 50.379, "p95_ms": 42.084}, "gstreamer_push": {"avg_ms": 0.863, "count": 94, "max_ms": 1.253, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.771, "count": 94, "max_ms": 46.064, "p95_ms": 42.184}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.239, "count": 94, "max_ms": 103.123, "p95_ms": 94.263}, "yolo11_640_fire_11class": {"avg_ms": 49.377, "count": 94, "max_ms": 71.654, "p95_ms": 70.25}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.688, "count": 94, "max_ms": 96.498, "p95_ms": 86.455}}, "truncated_count": 0}, "processing_total": {"avg_ms": 310.408, "count": 94, "max_ms": 474.366, "p95_ms": 386.178}, "read_interval": {"avg_ms": 319.977, "count": 93, "max_ms": 483.303, "p95_ms": 395.341}, "triton_wait": {"avg_ms": 73.352, "count": 94, "max_ms": 103.135, "p95_ms": 94.276}, "visualize": {"avg_ms": 11.889, "count": 94, "max_ms": 20.244, "p95_ms": 18.292}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.704, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 422, "tracking_summary": 11282}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10736.637, "limit_mb": 8796093022207.996}, "detail": {"pid": 83293, "restart_count": 440, "start_count": 441}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 403, "h264_decode_error_count": 1287, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "last_exit_at": "2026-07-01T06:17:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:17:05+09:00", "pid": 83293, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.973, "name": "ffmpeg", "pid": 83293, "rss_mb": 141.973, "state": "S (sleeping)", "threads": 106}, "restart_count": 440, "restart_count_per_hour": 21, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 441}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 390.254, "sample_count": 1098, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1353939, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:17:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.744, "last_frame_pushed_at": "2026-07-01T06:17:02+09:00", "last_frame_read_age_sec": 5.061, "last_frame_read_at": "2026-07-01T06:17:02+09:00", "mjpeg": {"age_sec": 4.723, "exists": true, "mtime": "2026-07-01T06:17:02.479551+09:00", "mtime_epoch": 1782854222.4795513, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169720}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:17:07+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5011.148, "name": "celery", "pid": 267, "rss_mb": 5011.148, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.706, "pushed": 2.706, "read": 2.672}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.606, "count": 80, "max_ms": 50.379, "p95_ms": 42.084}, "gstreamer_push": {"avg_ms": 0.866, "count": 81, "max_ms": 1.253, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.451, "count": 80, "max_ms": 46.064, "p95_ms": 42.184}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.601, "count": 80, "max_ms": 98.089, "p95_ms": 91.633}, "yolo11_640_fire_11class": {"avg_ms": 48.938, "count": 80, "max_ms": 70.833, "p95_ms": 68.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.363, "count": 80, "max_ms": 96.498, "p95_ms": 84.325}}, "truncated_count": 0}, "processing_total": {"avg_ms": 305.157, "count": 81, "max_ms": 474.366, "p95_ms": 372.958}, "read_interval": {"avg_ms": 314.659, "count": 80, "max_ms": 483.303, "p95_ms": 384.481}, "triton_wait": {"avg_ms": 72.654, "count": 80, "max_ms": 98.101, "p95_ms": 91.645}, "visualize": {"avg_ms": 11.048, "count": 80, "max_ms": 16.749, "p95_ms": 15.203}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 422, "tracking_summary": 11286}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10626.402, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 403, "h264_decode_error_count": 1287, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "last_exit_at": "2026-07-01T06:19:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.581, "last_started_at": "2026-07-01T06:17:05+09:00", "pid": 83293, "process": {"exists": false, "pid": 83293}, "restart_count": 440, "restart_count_per_hour": 20, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 441}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 404.715, "sample_count": 968, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1354359, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:19:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:19:09+09:00", "last_frame_read_age_sec": 0.389, "last_frame_read_at": "2026-07-01T06:19:09+09:00", "mjpeg": {"age_sec": 0.38, "exists": true, "mtime": "2026-07-01T06:19:09.402199+09:00", "mtime_epoch": 1782854349.4021988, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178466}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:19:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5011.406, "name": "celery", "pid": 267, "rss_mb": 5011.406, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.878, "pushed": 2.878, "read": 2.845}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.879, "count": 85, "max_ms": 42.644, "p95_ms": 41.649}, "gstreamer_push": {"avg_ms": 0.867, "count": 86, "max_ms": 1.366, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.795, "count": 85, "max_ms": 47.518, "p95_ms": 43.072}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.618, "count": 85, "max_ms": 100.07, "p95_ms": 95.756}, "yolo11_640_fire_11class": {"avg_ms": 50.028, "count": 85, "max_ms": 69.485, "p95_ms": 66.478}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.676, "count": 85, "max_ms": 92.126, "p95_ms": 88.211}}, "truncated_count": 0}, "processing_total": {"avg_ms": 342.057, "count": 86, "max_ms": 442.64, "p95_ms": 419.258}, "read_interval": {"avg_ms": 350.313, "count": 85, "max_ms": 450.762, "p95_ms": 426.184}, "triton_wait": {"avg_ms": 76.793, "count": 85, "max_ms": 100.083, "p95_ms": 95.768}, "visualize": {"avg_ms": 10.815, "count": 85, "max_ms": 14.121, "p95_ms": 13.296}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 422, "tracking_summary": 11286}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10626.395, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 403, "h264_decode_error_count": 1287, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "last_exit_at": "2026-07-01T06:19:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.584, "last_started_at": "2026-07-01T06:17:05+09:00", "pid": 83293, "process": {"exists": false, "pid": 83293}, "restart_count": 440, "restart_count_per_hour": 20, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 441}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 404.715, "sample_count": 968, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1354359, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:19:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T06:19:09+09:00", "last_frame_read_age_sec": 0.392, "last_frame_read_at": "2026-07-01T06:19:09+09:00", "mjpeg": {"age_sec": 0.383, "exists": true, "mtime": "2026-07-01T06:19:09.402199+09:00", "mtime_epoch": 1782854349.4021988, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178466}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:19:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5011.406, "name": "celery", "pid": 267, "rss_mb": 5011.406, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.878, "pushed": 2.878, "read": 2.844}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.879, "count": 85, "max_ms": 42.644, "p95_ms": 41.649}, "gstreamer_push": {"avg_ms": 0.867, "count": 86, "max_ms": 1.366, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.795, "count": 85, "max_ms": 47.518, "p95_ms": 43.072}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.618, "count": 85, "max_ms": 100.07, "p95_ms": 95.756}, "yolo11_640_fire_11class": {"avg_ms": 50.028, "count": 85, "max_ms": 69.485, "p95_ms": 66.478}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.676, "count": 85, "max_ms": 92.126, "p95_ms": 88.211}}, "truncated_count": 0}, "processing_total": {"avg_ms": 342.057, "count": 86, "max_ms": 442.64, "p95_ms": 419.258}, "read_interval": {"avg_ms": 350.313, "count": 85, "max_ms": 450.762, "p95_ms": 426.184}, "triton_wait": {"avg_ms": 76.793, "count": 85, "max_ms": 100.083, "p95_ms": 95.768}, "visualize": {"avg_ms": 10.815, "count": 85, "max_ms": 14.121, "p95_ms": 13.296}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 422, "tracking_summary": 11286}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10735.395, "limit_mb": 8796093022207.996}, "detail": {"pid": 83400, "restart_count": 441, "start_count": 442}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 403, "h264_decode_error_count": 1287, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:14:57+09:00", "last_error_summary": "[h264 @ 0x55ec39542ac0] error while decoding MB 69 34, bytestream -9", "last_exit_at": "2026-07-01T06:19:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:19:12+09:00", "pid": 83400, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.754, "name": "ffmpeg", "pid": 83400, "rss_mb": 138.754, "state": "S (sleeping)", "threads": 106}, "restart_count": 441, "restart_count_per_hour": 21, "rtsp_timeout_count": 384, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 442}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 404.715, "sample_count": 953, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1354359, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:19:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.379, "last_frame_pushed_at": "2026-07-01T06:19:09+09:00", "last_frame_read_age_sec": 4.766, "last_frame_read_at": "2026-07-01T06:19:09+09:00", "mjpeg": {"age_sec": 4.365, "exists": true, "mtime": "2026-07-01T06:19:09.794207+09:00", "mtime_epoch": 1782854349.7942069, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174666}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:19:14+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5011.406, "name": "celery", "pid": 267, "rss_mb": 5011.406, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 70, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.389, "pushed": 2.389, "read": 2.355}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 37.253, "count": 71, "max_ms": 42.644, "p95_ms": 41.676}, "gstreamer_push": {"avg_ms": 0.87, "count": 71, "max_ms": 1.366, "p95_ms": 0.919}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.463, "count": 71, "max_ms": 47.518, "p95_ms": 41.39}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.293, "count": 71, "max_ms": 100.07, "p95_ms": 95.855}, "yolo11_640_fire_11class": {"avg_ms": 51.011, "count": 71, "max_ms": 69.485, "p95_ms": 66.788}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.117, "count": 71, "max_ms": 92.126, "p95_ms": 88.563}}, "truncated_count": 0}, "processing_total": {"avg_ms": 352.714, "count": 71, "max_ms": 442.64, "p95_ms": 426.765}, "read_interval": {"avg_ms": 360.845, "count": 70, "max_ms": 450.762, "p95_ms": 434.76}, "triton_wait": {"avg_ms": 77.499, "count": 71, "max_ms": 100.083, "p95_ms": 95.868}, "visualize": {"avg_ms": 11.092, "count": 71, "max_ms": 14.121, "p95_ms": 13.438}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.722, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 425, "tracking_summary": 11291}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10635.438, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c2bbcf13c0] cabac decode of qscale diff failed at 48 44 [h264 @ 0x55c2bbcf13c0] error while decoding MB 48 44, bytestream 234 [in#0/rtsp @ 0x55c2bb188e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c2bb188e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 404, "h264_decode_error_count": 1289, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:22:06+09:00", "last_error_summary": "[h264 @ 0x55c2bbcf13c0] cabac decode of qscale diff failed at 48 44 [h264 @ 0x55c2bbcf13c0] error while decoding MB 48 44, bytestream 234 [in#0/rtsp @ 0x55c2bb188e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c2bb188e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:19:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 173.98, "last_started_at": "2026-07-01T06:19:12+09:00", "pid": 83400, "process": {"exists": false, "pid": 83400}, "restart_count": 441, "restart_count_per_hour": 21, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 442}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 378.15, "sample_count": 1051, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1354990, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:22:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T06:22:06+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T06:22:06+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T06:22:06.097884+09:00", "mtime_epoch": 1782854526.0978842, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187286}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:22:06+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5011.664, "name": "celery", "pid": 267, "rss_mb": 5011.664, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 62, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.068, "pushed": 2.068, "read": 2.068}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.282, "count": 61, "max_ms": 36.438, "p95_ms": 33.801}, "gstreamer_push": {"avg_ms": 0.931, "count": 62, "max_ms": 1.149, "p95_ms": 1.046}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.267, "count": 61, "max_ms": 43.87, "p95_ms": 37.951}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.12, "count": 61, "max_ms": 94.011, "p95_ms": 86.661}, "yolo11_640_fire_11class": {"avg_ms": 50.954, "count": 61, "max_ms": 70.502, "p95_ms": 69.271}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.191, "count": 61, "max_ms": 85.194, "p95_ms": 80.411}}, "truncated_count": 0}, "processing_total": {"avg_ms": 182.121, "count": 62, "max_ms": 288.293, "p95_ms": 221.608}, "read_interval": {"avg_ms": 487.472, "count": 62, "max_ms": 10136.017, "p95_ms": 266.654}, "triton_wait": {"avg_ms": 67.132, "count": 61, "max_ms": 94.023, "p95_ms": 86.673}, "visualize": {"avg_ms": 6.023, "count": 61, "max_ms": 8.717, "p95_ms": 7.418}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 425, "tracking_summary": 11291}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10634.852, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:22:06+09:00", "last_error_summary": "[h264 @ 0x55c2bbcf13c0] cabac decode of qscale diff failed at 48 44 [h264 @ 0x55c2bbcf13c0] error while decoding MB 48 44, bytestream 234 [in#0/rtsp @ 0x55c2bb188e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c2bb188e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 404, "h264_decode_error_count": 1289, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:22:06+09:00", "last_error_summary": "[h264 @ 0x55c2bbcf13c0] cabac decode of qscale diff failed at 48 44 [h264 @ 0x55c2bbcf13c0] error while decoding MB 48 44, bytestream 234 [in#0/rtsp @ 0x55c2bb188e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c2bb188e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:22:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 174.09, "last_started_at": "2026-07-01T06:19:12+09:00", "pid": 83400, "process": {"exists": false, "pid": 83400}, "restart_count": 441, "restart_count_per_hour": 21, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 442}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 376.501, "sample_count": 1052, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1354991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:22:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:22:06+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-01T06:22:06+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-01T06:22:06.097884+09:00", "mtime_epoch": 1782854526.0978842, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187286}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:22:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5011.664, "name": "celery", "pid": 267, "rss_mb": 5011.664, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.079, "pushed": 2.079, "read": 2.045}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.259, "count": 62, "max_ms": 36.438, "p95_ms": 33.801}, "gstreamer_push": {"avg_ms": 0.933, "count": 62, "max_ms": 1.149, "p95_ms": 1.046}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.252, "count": 62, "max_ms": 43.87, "p95_ms": 37.951}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.966, "count": 62, "max_ms": 94.011, "p95_ms": 86.661}, "yolo11_640_fire_11class": {"avg_ms": 50.823, "count": 62, "max_ms": 70.502, "p95_ms": 69.271}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.056, "count": 62, "max_ms": 85.194, "p95_ms": 80.411}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.975, "count": 62, "max_ms": 258.906, "p95_ms": 217.613}, "read_interval": {"avg_ms": 490.592, "count": 61, "max_ms": 10136.017, "p95_ms": 234.244}, "triton_wait": {"avg_ms": 66.979, "count": 62, "max_ms": 94.023, "p95_ms": 86.673}, "visualize": {"avg_ms": 6.013, "count": 62, "max_ms": 8.717, "p95_ms": 7.418}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.827, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 425, "tracking_summary": 11291}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10634.969, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 404, "h264_decode_error_count": 1289, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:22:06+09:00", "last_error_summary": "[h264 @ 0x55c2bbcf13c0] cabac decode of qscale diff failed at 48 44 [h264 @ 0x55c2bbcf13c0] error while decoding MB 48 44, bytestream 234 [in#0/rtsp @ 0x55c2bb188e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c2bb188e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:22:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 174.093, "last_started_at": "2026-07-01T06:19:12+09:00", "pid": 83400, "process": {"exists": false, "pid": 83400}, "restart_count": 441, "restart_count_per_hour": 21, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 442}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 376.501, "sample_count": 1052, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1354991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:22:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T06:22:06+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-01T06:22:06+09:00", "mjpeg": {"age_sec": 0.153, "exists": true, "mtime": "2026-07-01T06:22:06.097884+09:00", "mtime_epoch": 1782854526.0978842, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187286}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:22:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5011.664, "name": "celery", "pid": 267, "rss_mb": 5011.664, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.078, "pushed": 2.078, "read": 2.045}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.259, "count": 62, "max_ms": 36.438, "p95_ms": 33.801}, "gstreamer_push": {"avg_ms": 0.933, "count": 62, "max_ms": 1.149, "p95_ms": 1.046}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.252, "count": 62, "max_ms": 43.87, "p95_ms": 37.951}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.966, "count": 62, "max_ms": 94.011, "p95_ms": 86.661}, "yolo11_640_fire_11class": {"avg_ms": 50.823, "count": 62, "max_ms": 70.502, "p95_ms": 69.271}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.056, "count": 62, "max_ms": 85.194, "p95_ms": 80.411}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.975, "count": 62, "max_ms": 258.906, "p95_ms": 217.613}, "read_interval": {"avg_ms": 490.592, "count": 61, "max_ms": 10136.017, "p95_ms": 234.244}, "triton_wait": {"avg_ms": 66.979, "count": 62, "max_ms": 94.023, "p95_ms": 86.673}, "visualize": {"avg_ms": 6.013, "count": 62, "max_ms": 8.717, "p95_ms": 7.418}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.83, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 425, "tracking_summary": 11291}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10745.656, "limit_mb": 8796093022207.996}, "detail": {"pid": 83567, "restart_count": 442, "start_count": 443}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 404, "h264_decode_error_count": 1289, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:22:06+09:00", "last_error_summary": "[h264 @ 0x55c2bbcf13c0] cabac decode of qscale diff failed at 48 44 [h264 @ 0x55c2bbcf13c0] error while decoding MB 48 44, bytestream 234 [in#0/rtsp @ 0x55c2bb188e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c2bb188e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:22:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:22:08+09:00", "pid": 83567, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.312, "name": "ffmpeg", "pid": 83567, "rss_mb": 141.312, "state": "S (sleeping)", "threads": 106}, "restart_count": 442, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 443}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 378.15, "sample_count": 1038, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1354991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:22:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.603, "last_frame_pushed_at": "2026-07-01T06:22:06+09:00", "last_frame_read_age_sec": 4.759, "last_frame_read_at": "2026-07-01T06:22:06+09:00", "mjpeg": {"age_sec": 4.586, "exists": true, "mtime": "2026-07-01T06:22:06.261888+09:00", "mtime_epoch": 1782854526.2618878, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187606}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:22:10+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5011.664, "name": "celery", "pid": 267, "rss_mb": 5011.664, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 41, "pushed_frames": 41, "read_frames": 40, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.37, "pushed": 1.37, "read": 1.337}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 29.981, "count": 40, "max_ms": 36.438, "p95_ms": 34.559}, "gstreamer_push": {"avg_ms": 0.973, "count": 41, "max_ms": 1.149, "p95_ms": 1.075}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.205, "count": 40, "max_ms": 43.87, "p95_ms": 38.842}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.504, "count": 40, "max_ms": 87.44, "p95_ms": 81.227}, "yolo11_640_fire_11class": {"avg_ms": 48.607, "count": 40, "max_ms": 68.559, "p95_ms": 65.132}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.298, "count": 40, "max_ms": 83.376, "p95_ms": 74.009}}, "truncated_count": 0}, "processing_total": {"avg_ms": 164.848, "count": 41, "max_ms": 196.681, "p95_ms": 185.328}, "read_interval": {"avg_ms": 633.994, "count": 40, "max_ms": 10136.017, "p95_ms": 204.267}, "triton_wait": {"avg_ms": 64.517, "count": 40, "max_ms": 87.45, "p95_ms": 81.24}, "visualize": {"avg_ms": 5.407, "count": 40, "max_ms": 7.362, "p95_ms": 6.042}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 426, "tracking_summary": 11295}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10641.418, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f3c70ff340] error while decoding MB 52 29, bytestream -54", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 405, "h264_decode_error_count": 1290, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:23:59+09:00", "last_error_summary": "[h264 @ 0x55f3c70ff340] error while decoding MB 52 29, bytestream -54", "last_exit_at": "2026-07-01T06:22:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.846, "last_started_at": "2026-07-01T06:22:08+09:00", "pid": 83567, "process": {"exists": false, "pid": 83567}, "restart_count": 442, "restart_count_per_hour": 21, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 443}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 368.46, "sample_count": 1089, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355422, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:23:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.045, "last_frame_pushed_at": "2026-07-01T06:23:59+09:00", "last_frame_read_age_sec": 0.037, "last_frame_read_at": "2026-07-01T06:23:59+09:00", "mjpeg": {"age_sec": 0.027, "exists": true, "mtime": "2026-07-01T06:23:59.665351+09:00", "mtime_epoch": 1782854639.6653512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180093}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:23:59+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5011.664, "name": "celery", "pid": 267, "rss_mb": 5011.664, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.716, "pushed": 3.716, "read": 3.716}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.202, "count": 110, "max_ms": 53.33, "p95_ms": 39.274}, "gstreamer_push": {"avg_ms": 0.867, "count": 111, "max_ms": 1.314, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.843, "count": 110, "max_ms": 66.906, "p95_ms": 47.327}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.783, "count": 110, "max_ms": 100.487, "p95_ms": 94.914}, "yolo11_640_fire_11class": {"avg_ms": 48.766, "count": 110, "max_ms": 66.695, "p95_ms": 63.994}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.949, "count": 110, "max_ms": 91.271, "p95_ms": 82.173}}, "truncated_count": 0}, "processing_total": {"avg_ms": 260.942, "count": 111, "max_ms": 373.536, "p95_ms": 345.676}, "read_interval": {"avg_ms": 270.93, "count": 111, "max_ms": 383.357, "p95_ms": 356.257}, "triton_wait": {"avg_ms": 73.133, "count": 110, "max_ms": 100.499, "p95_ms": 94.926}, "visualize": {"avg_ms": 7.434, "count": 110, "max_ms": 10.964, "p95_ms": 9.451}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.875, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 426, "tracking_summary": 11295}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10640.984, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:23:59+09:00", "last_error_summary": "[h264 @ 0x55f3c70ff340] error while decoding MB 52 29, bytestream -54", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 405, "h264_decode_error_count": 1290, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:23:59+09:00", "last_error_summary": "[h264 @ 0x55f3c70ff340] error while decoding MB 52 29, bytestream -54", "last_exit_at": "2026-07-01T06:23:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.067, "last_started_at": "2026-07-01T06:22:08+09:00", "pid": 83567, "process": {"exists": false, "pid": 83567}, "restart_count": 442, "restart_count_per_hour": 21, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 443}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 366.996, "sample_count": 1089, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355423, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:23:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:23:59+09:00", "last_frame_read_age_sec": 0.258, "last_frame_read_at": "2026-07-01T06:23:59+09:00", "mjpeg": {"age_sec": 0.248, "exists": true, "mtime": "2026-07-01T06:23:59.665351+09:00", "mtime_epoch": 1782854639.6653512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180093}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:23:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5011.664, "name": "celery", "pid": 267, "rss_mb": 5011.664, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.721, "pushed": 3.721, "read": 3.688}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.167, "count": 111, "max_ms": 53.33, "p95_ms": 39.274}, "gstreamer_push": {"avg_ms": 0.868, "count": 111, "max_ms": 1.314, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.813, "count": 111, "max_ms": 66.906, "p95_ms": 47.327}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.953, "count": 111, "max_ms": 100.487, "p95_ms": 94.914}, "yolo11_640_fire_11class": {"avg_ms": 48.859, "count": 111, "max_ms": 66.695, "p95_ms": 63.994}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.052, "count": 111, "max_ms": 91.271, "p95_ms": 82.173}}, "truncated_count": 0}, "processing_total": {"avg_ms": 261.103, "count": 111, "max_ms": 373.536, "p95_ms": 345.676}, "read_interval": {"avg_ms": 271.147, "count": 110, "max_ms": 383.357, "p95_ms": 356.257}, "triton_wait": {"avg_ms": 73.3, "count": 111, "max_ms": 100.499, "p95_ms": 94.926}, "visualize": {"avg_ms": 7.431, "count": 111, "max_ms": 10.964, "p95_ms": 9.451}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.83, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 426, "tracking_summary": 11295}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10640.953, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 405, "h264_decode_error_count": 1290, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:23:59+09:00", "last_error_summary": "[h264 @ 0x55f3c70ff340] error while decoding MB 52 29, bytestream -54", "last_exit_at": "2026-07-01T06:23:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.071, "last_started_at": "2026-07-01T06:22:08+09:00", "pid": 83567, "process": {"exists": false, "pid": 83567}, "restart_count": 442, "restart_count_per_hour": 21, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 443}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 366.996, "sample_count": 1089, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355423, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:23:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T06:23:59+09:00", "last_frame_read_age_sec": 0.262, "last_frame_read_at": "2026-07-01T06:23:59+09:00", "mjpeg": {"age_sec": 0.252, "exists": true, "mtime": "2026-07-01T06:23:59.665351+09:00", "mtime_epoch": 1782854639.6653512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180093}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:23:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5011.664, "name": "celery", "pid": 267, "rss_mb": 5011.664, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.721, "pushed": 3.721, "read": 3.687}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.167, "count": 111, "max_ms": 53.33, "p95_ms": 39.274}, "gstreamer_push": {"avg_ms": 0.868, "count": 111, "max_ms": 1.314, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.813, "count": 111, "max_ms": 66.906, "p95_ms": 47.327}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.953, "count": 111, "max_ms": 100.487, "p95_ms": 94.914}, "yolo11_640_fire_11class": {"avg_ms": 48.859, "count": 111, "max_ms": 66.695, "p95_ms": 63.994}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.052, "count": 111, "max_ms": 91.271, "p95_ms": 82.173}}, "truncated_count": 0}, "processing_total": {"avg_ms": 261.103, "count": 111, "max_ms": 373.536, "p95_ms": 345.676}, "read_interval": {"avg_ms": 271.147, "count": 110, "max_ms": 383.357, "p95_ms": 356.257}, "triton_wait": {"avg_ms": 73.3, "count": 111, "max_ms": 100.499, "p95_ms": 94.926}, "visualize": {"avg_ms": 7.431, "count": 111, "max_ms": 10.964, "p95_ms": 9.451}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.833, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 426, "tracking_summary": 11295}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10751.637, "limit_mb": 8796093022207.996}, "detail": {"pid": 83734, "restart_count": 443, "start_count": 444}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 405, "h264_decode_error_count": 1290, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:23:59+09:00", "last_error_summary": "[h264 @ 0x55f3c70ff340] error while decoding MB 52 29, bytestream -54", "last_exit_at": "2026-07-01T06:24:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:24:02+09:00", "pid": 83734, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.312, "name": "ffmpeg", "pid": 83734, "rss_mb": 141.312, "state": "S (sleeping)", "threads": 106}, "restart_count": 443, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 444}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 349.603, "sample_count": 1078, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355423, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:23:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.483, "last_frame_pushed_at": "2026-07-01T06:23:59+09:00", "last_frame_read_age_sec": 4.739, "last_frame_read_at": "2026-07-01T06:23:59+09:00", "mjpeg": {"age_sec": 4.465, "exists": true, "mtime": "2026-07-01T06:23:59.929344+09:00", "mtime_epoch": 1782854639.929344, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177857}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:24:04+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5011.664, "name": "celery", "pid": 267, "rss_mb": 5011.664, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.203, "pushed": 3.203, "read": 3.17}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.793, "count": 95, "max_ms": 44.923, "p95_ms": 38.881}, "gstreamer_push": {"avg_ms": 0.861, "count": 96, "max_ms": 0.957, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 37.116, "count": 95, "max_ms": 66.906, "p95_ms": 52.012}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.019, "count": 95, "max_ms": 100.487, "p95_ms": 93.125}, "yolo11_640_fire_11class": {"avg_ms": 48.999, "count": 95, "max_ms": 66.695, "p95_ms": 63.994}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.985, "count": 95, "max_ms": 91.271, "p95_ms": 82.173}}, "truncated_count": 0}, "processing_total": {"avg_ms": 258.996, "count": 96, "max_ms": 373.536, "p95_ms": 347.265}, "read_interval": {"avg_ms": 268.97, "count": 95, "max_ms": 383.357, "p95_ms": 357.877}, "triton_wait": {"avg_ms": 73.422, "count": 95, "max_ms": 100.499, "p95_ms": 93.137}, "visualize": {"avg_ms": 7.328, "count": 95, "max_ms": 10.964, "p95_ms": 9.207}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 428, "tracking_summary": 11297}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10800.695, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55cf3e401f80] error while decoding MB 23 30, bytestream -19", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 406, "h264_decode_error_count": 1291, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:26:30+09:00", "last_error_summary": "[h264 @ 0x55cf3e401f80] error while decoding MB 23 30, bytestream -19", "last_exit_at": "2026-07-01T06:24:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 147.953, "last_started_at": "2026-07-01T06:24:02+09:00", "pid": 83734, "process": {"exists": false, "pid": 83734}, "restart_count": 443, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 444}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 655.152, "sample_count": 745, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355657, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:26:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T06:26:30+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T06:26:30+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T06:26:30.306098+09:00", "mtime_epoch": 1782854790.3060977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190565}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:26:30+09:00", "process": {"exists": true, "fd_count": 139, "hwm_mb": 5011.922, "name": "celery", "pid": 267, "rss_mb": 5011.922, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": 18.585, "recorders": [{"age_sec": 18.585, "frame_count": 80, "is_recording": true, "is_stopping": false, "pid": 83841, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.621, "name": "ffmpeg", "pid": 83841, "rss_mb": 186.621, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 20, "returncode": null, "running": true, "started_at": "2026-07-01T06:26:11+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_062611_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [83841], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 49, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.649, "pushed": 1.649, "read": 1.649}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 54.277, "count": 48, "max_ms": 59.958, "p95_ms": 58.458}, "gstreamer_push": {"avg_ms": 0.852, "count": 49, "max_ms": 0.927, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.112, "count": 48, "max_ms": 67.924, "p95_ms": 63.215}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.164, "count": 48, "max_ms": 99.389, "p95_ms": 93.43}, "yolo11_640_fire_11class": {"avg_ms": 46.689, "count": 48, "max_ms": 65.218, "p95_ms": 59.918}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.571, "count": 48, "max_ms": 86.87, "p95_ms": 82.586}}, "truncated_count": 0}, "processing_total": {"avg_ms": 608.22, "count": 49, "max_ms": 1365.855, "p95_ms": 677.949}, "read_interval": {"avg_ms": 617.732, "count": 49, "max_ms": 1375.727, "p95_ms": 688.494}, "triton_wait": {"avg_ms": 76.183, "count": 48, "max_ms": 99.401, "p95_ms": 93.443}, "visualize": {"avg_ms": 13.298, "count": 48, "max_ms": 16.774, "p95_ms": 16.517}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.722, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 428, "tracking_summary": 11297}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10799.605, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:26:30+09:00", "last_error_summary": "[h264 @ 0x55cf3e401f80] error while decoding MB 23 30, bytestream -19", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 406, "h264_decode_error_count": 1291, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:26:30+09:00", "last_error_summary": "[h264 @ 0x55cf3e401f80] error while decoding MB 23 30, bytestream -19", "last_exit_at": "2026-07-01T06:26:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.459, "last_started_at": "2026-07-01T06:24:02+09:00", "pid": 83734, "process": {"exists": false, "pid": 83734}, "restart_count": 443, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 444}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 655.152, "sample_count": 745, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:26:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T06:26:30+09:00", "last_frame_read_age_sec": 0.556, "last_frame_read_at": "2026-07-01T06:26:30+09:00", "mjpeg": {"age_sec": 0.546, "exists": true, "mtime": "2026-07-01T06:26:30.306098+09:00", "mtime_epoch": 1782854790.3060977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190565}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:26:30+09:00", "process": {"exists": true, "fd_count": 138, "hwm_mb": 5011.922, "name": "celery", "pid": 267, "rss_mb": 5011.922, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": 19.09, "recorders": [{"age_sec": 19.09, "frame_count": 81, "is_recording": true, "is_stopping": false, "pid": 83841, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.621, "name": "ffmpeg", "pid": 83841, "rss_mb": 186.621, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 19, "returncode": null, "running": true, "started_at": "2026-07-01T06:26:11+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_062611_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [83841], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.654, "pushed": 1.654, "read": 1.62}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 54.254, "count": 48, "max_ms": 59.958, "p95_ms": 58.458}, "gstreamer_push": {"avg_ms": 0.853, "count": 49, "max_ms": 0.927, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.447, "count": 48, "max_ms": 67.924, "p95_ms": 53.044}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.164, "count": 48, "max_ms": 99.389, "p95_ms": 93.43}, "yolo11_640_fire_11class": {"avg_ms": 46.488, "count": 48, "max_ms": 65.218, "p95_ms": 59.918}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.407, "count": 48, "max_ms": 86.87, "p95_ms": 82.586}}, "truncated_count": 0}, "processing_total": {"avg_ms": 607.277, "count": 49, "max_ms": 1365.855, "p95_ms": 677.949}, "read_interval": {"avg_ms": 617.977, "count": 48, "max_ms": 1375.727, "p95_ms": 688.494}, "triton_wait": {"avg_ms": 76.182, "count": 48, "max_ms": 99.401, "p95_ms": 93.443}, "visualize": {"avg_ms": 13.365, "count": 48, "max_ms": 16.774, "p95_ms": 16.517}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.624, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 428, "tracking_summary": 11297}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10799.723, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 406, "h264_decode_error_count": 1291, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:26:30+09:00", "last_error_summary": "[h264 @ 0x55cf3e401f80] error while decoding MB 23 30, bytestream -19", "last_exit_at": "2026-07-01T06:26:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.462, "last_started_at": "2026-07-01T06:24:02+09:00", "pid": 83734, "process": {"exists": false, "pid": 83734}, "restart_count": 443, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 444}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 655.152, "sample_count": 745, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:26:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T06:26:30+09:00", "last_frame_read_age_sec": 0.559, "last_frame_read_at": "2026-07-01T06:26:30+09:00", "mjpeg": {"age_sec": 0.549, "exists": true, "mtime": "2026-07-01T06:26:30.306098+09:00", "mtime_epoch": 1782854790.3060977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190565}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:26:30+09:00", "process": {"exists": true, "fd_count": 138, "hwm_mb": 5011.922, "name": "celery", "pid": 267, "rss_mb": 5011.922, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": 19.093, "recorders": [{"age_sec": 19.093, "frame_count": 81, "is_recording": true, "is_stopping": false, "pid": 83841, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.621, "name": "ffmpeg", "pid": 83841, "rss_mb": 186.621, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 19, "returncode": null, "running": true, "started_at": "2026-07-01T06:26:11+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_062611_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [83841], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.654, "pushed": 1.654, "read": 1.62}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 54.254, "count": 48, "max_ms": 59.958, "p95_ms": 58.458}, "gstreamer_push": {"avg_ms": 0.853, "count": 49, "max_ms": 0.927, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.447, "count": 48, "max_ms": 67.924, "p95_ms": 53.044}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.164, "count": 48, "max_ms": 99.389, "p95_ms": 93.43}, "yolo11_640_fire_11class": {"avg_ms": 46.488, "count": 48, "max_ms": 65.218, "p95_ms": 59.918}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.407, "count": 48, "max_ms": 86.87, "p95_ms": 82.586}}, "truncated_count": 0}, "processing_total": {"avg_ms": 607.277, "count": 49, "max_ms": 1365.855, "p95_ms": 677.949}, "read_interval": {"avg_ms": 617.977, "count": 48, "max_ms": 1375.727, "p95_ms": 688.494}, "triton_wait": {"avg_ms": 76.182, "count": 48, "max_ms": 99.401, "p95_ms": 93.443}, "visualize": {"avg_ms": 13.365, "count": 48, "max_ms": 16.774, "p95_ms": 16.517}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.627, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 428, "tracking_summary": 11297}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10909.125, "limit_mb": 8796093022207.996}, "detail": {"pid": 83901, "restart_count": 444, "start_count": 445}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 406, "h264_decode_error_count": 1291, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:26:30+09:00", "last_error_summary": "[h264 @ 0x55cf3e401f80] error while decoding MB 23 30, bytestream -19", "last_exit_at": "2026-07-01T06:26:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:26:32+09:00", "pid": 83901, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.566, "name": "ffmpeg", "pid": 83901, "rss_mb": 138.566, "state": "S (sleeping)", "threads": 106}, "restart_count": 444, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 445}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 655.152, "sample_count": 734, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:26:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.063, "last_frame_pushed_at": "2026-07-01T06:26:30+09:00", "last_frame_read_age_sec": 4.615, "last_frame_read_at": "2026-07-01T06:26:30+09:00", "mjpeg": {"age_sec": 4.046, "exists": true, "mtime": "2026-07-01T06:26:30.866088+09:00", "mtime_epoch": 1782854790.8660884, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190238}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:26:34+09:00", "process": {"exists": true, "fd_count": 139, "hwm_mb": 5011.922, "name": "celery", "pid": 267, "rss_mb": 5011.922, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": 23.15, "recorders": [{"age_sec": 23.15, "frame_count": 81, "is_recording": true, "is_stopping": false, "pid": 83841, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.621, "name": "ffmpeg", "pid": 83841, "rss_mb": 186.621, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 19, "returncode": null, "running": true, "started_at": "2026-07-01T06:26:11+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_062611_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [83841], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.435, "pushed": 1.435, "read": 1.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 53.82, "count": 42, "max_ms": 59.768, "p95_ms": 58.41}, "gstreamer_push": {"avg_ms": 0.853, "count": 43, "max_ms": 0.913, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 37.51, "count": 42, "max_ms": 48.49, "p95_ms": 45.697}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.279, "count": 42, "max_ms": 99.389, "p95_ms": 93.43}, "yolo11_640_fire_11class": {"avg_ms": 46.466, "count": 42, "max_ms": 65.218, "p95_ms": 59.918}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.607, "count": 42, "max_ms": 86.87, "p95_ms": 82.586}}, "truncated_count": 0}, "processing_total": {"avg_ms": 606.675, "count": 43, "max_ms": 1365.855, "p95_ms": 677.949}, "read_interval": {"avg_ms": 617.554, "count": 42, "max_ms": 1375.727, "p95_ms": 688.494}, "triton_wait": {"avg_ms": 76.292, "count": 42, "max_ms": 99.401, "p95_ms": 93.443}, "visualize": {"avg_ms": 13.64, "count": 42, "max_ms": 16.774, "p95_ms": 16.517}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 434, "tracking_summary": 11298}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10658.395, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562571b00300] error while decoding MB 69 41, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 407, "h264_decode_error_count": 1292, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:28:30+09:00", "last_error_summary": "[h264 @ 0x562571b00300] error while decoding MB 69 41, bytestream -7", "last_exit_at": "2026-07-01T06:26:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 117.969, "last_started_at": "2026-07-01T06:26:32+09:00", "pid": 83901, "process": {"exists": false, "pid": 83901}, "restart_count": 444, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 445}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 867.508, "sample_count": 495, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355811, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:28:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T06:28:30+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T06:28:30+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T06:28:30.836490+09:00", "mtime_epoch": 1782854910.8364904, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189726}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:28:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5011.922, "name": "celery", "pid": 267, "rss_mb": 5011.922, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 44, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.474, "pushed": 1.474, "read": 1.474}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 61.335, "count": 43, "max_ms": 72.567, "p95_ms": 70.333}, "gstreamer_push": {"avg_ms": 0.855, "count": 44, "max_ms": 0.907, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.996, "count": 43, "max_ms": 63.572, "p95_ms": 51.749}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.709, "count": 43, "max_ms": 94.815, "p95_ms": 89.736}, "yolo11_640_fire_11class": {"avg_ms": 45.774, "count": 43, "max_ms": 65.041, "p95_ms": 62.874}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.88, "count": 43, "max_ms": 84.481, "p95_ms": 81.683}}, "truncated_count": 0}, "processing_total": {"avg_ms": 680.734, "count": 44, "max_ms": 782.806, "p95_ms": 752.69}, "read_interval": {"avg_ms": 689.917, "count": 44, "max_ms": 792.034, "p95_ms": 761.369}, "triton_wait": {"avg_ms": 70.219, "count": 43, "max_ms": 94.828, "p95_ms": 89.748}, "visualize": {"avg_ms": 11.75, "count": 43, "max_ms": 14.784, "p95_ms": 14.023}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.854, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 434, "tracking_summary": 11298}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10657.316, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:28:30+09:00", "last_error_summary": "[h264 @ 0x562571b00300] error while decoding MB 69 41, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 407, "h264_decode_error_count": 1292, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:28:30+09:00", "last_error_summary": "[h264 @ 0x562571b00300] error while decoding MB 69 41, bytestream -7", "last_exit_at": "2026-07-01T06:28:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.569, "last_started_at": "2026-07-01T06:26:32+09:00", "pid": 83901, "process": {"exists": false, "pid": 83901}, "restart_count": 444, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 445}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 867.508, "sample_count": 495, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355812, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:28:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-01T06:28:31+09:00", "last_frame_read_age_sec": 0.652, "last_frame_read_at": "2026-07-01T06:28:30+09:00", "mjpeg": {"age_sec": 0.644, "exists": true, "mtime": "2026-07-01T06:28:30.836490+09:00", "mtime_epoch": 1782854910.8364904, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189726}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:28:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5011.922, "name": "celery", "pid": 267, "rss_mb": 5011.922, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.475, "pushed": 1.475, "read": 1.441}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 61.508, "count": 43, "max_ms": 72.567, "p95_ms": 70.333}, "gstreamer_push": {"avg_ms": 0.856, "count": 44, "max_ms": 0.907, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.85, "count": 43, "max_ms": 63.572, "p95_ms": 51.749}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.525, "count": 43, "max_ms": 94.815, "p95_ms": 89.736}, "yolo11_640_fire_11class": {"avg_ms": 45.811, "count": 43, "max_ms": 65.041, "p95_ms": 62.874}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.812, "count": 43, "max_ms": 84.481, "p95_ms": 81.683}}, "truncated_count": 0}, "processing_total": {"avg_ms": 682.832, "count": 44, "max_ms": 782.806, "p95_ms": 752.69}, "read_interval": {"avg_ms": 692.832, "count": 43, "max_ms": 792.034, "p95_ms": 761.369}, "triton_wait": {"avg_ms": 70.034, "count": 43, "max_ms": 94.828, "p95_ms": 89.748}, "visualize": {"avg_ms": 11.745, "count": 43, "max_ms": 14.784, "p95_ms": 14.023}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.836, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 434, "tracking_summary": 11298}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10657.309, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 407, "h264_decode_error_count": 1292, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:28:30+09:00", "last_error_summary": "[h264 @ 0x562571b00300] error while decoding MB 69 41, bytestream -7", "last_exit_at": "2026-07-01T06:28:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.573, "last_started_at": "2026-07-01T06:26:32+09:00", "pid": 83901, "process": {"exists": false, "pid": 83901}, "restart_count": 444, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 445}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 867.508, "sample_count": 495, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355812, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:28:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T06:28:31+09:00", "last_frame_read_age_sec": 0.656, "last_frame_read_at": "2026-07-01T06:28:30+09:00", "mjpeg": {"age_sec": 0.647, "exists": true, "mtime": "2026-07-01T06:28:30.836490+09:00", "mtime_epoch": 1782854910.8364904, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189726}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:28:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5011.922, "name": "celery", "pid": 267, "rss_mb": 5011.922, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.475, "pushed": 1.475, "read": 1.441}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 61.508, "count": 43, "max_ms": 72.567, "p95_ms": 70.333}, "gstreamer_push": {"avg_ms": 0.856, "count": 44, "max_ms": 0.907, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.85, "count": 43, "max_ms": 63.572, "p95_ms": 51.749}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.525, "count": 43, "max_ms": 94.815, "p95_ms": 89.736}, "yolo11_640_fire_11class": {"avg_ms": 45.811, "count": 43, "max_ms": 65.041, "p95_ms": 62.874}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.812, "count": 43, "max_ms": 84.481, "p95_ms": 81.683}}, "truncated_count": 0}, "processing_total": {"avg_ms": 682.832, "count": 44, "max_ms": 782.806, "p95_ms": 752.69}, "read_interval": {"avg_ms": 692.832, "count": 43, "max_ms": 792.034, "p95_ms": 761.369}, "triton_wait": {"avg_ms": 70.034, "count": 43, "max_ms": 94.828, "p95_ms": 89.748}, "visualize": {"avg_ms": 11.745, "count": 43, "max_ms": 14.784, "p95_ms": 14.023}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.84, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 434, "tracking_summary": 11298}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10765.543, "limit_mb": 8796093022207.996}, "detail": {"pid": 84008, "restart_count": 445, "start_count": 446}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 407, "h264_decode_error_count": 1292, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:28:30+09:00", "last_error_summary": "[h264 @ 0x562571b00300] error while decoding MB 69 41, bytestream -7", "last_exit_at": "2026-07-01T06:28:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:28:33+09:00", "pid": 84008, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.84, "name": "ffmpeg", "pid": 84008, "rss_mb": 138.84, "state": "S (sleeping)", "threads": 106}, "restart_count": 445, "restart_count_per_hour": 23, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 446}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 871.446, "sample_count": 481, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1355812, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:28:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.19, "last_frame_pushed_at": "2026-07-01T06:28:31+09:00", "last_frame_read_age_sec": 4.839, "last_frame_read_at": "2026-07-01T06:28:30+09:00", "mjpeg": {"age_sec": 4.175, "exists": true, "mtime": "2026-07-01T06:28:31.492484+09:00", "mtime_epoch": 1782854911.4924836, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190486}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:28:35+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5011.922, "name": "celery", "pid": 267, "rss_mb": 5011.922, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 37, "pushed_frames": 37, "read_frames": 36, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.254, "pushed": 1.254, "read": 1.221}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 62.155, "count": 37, "max_ms": 72.567, "p95_ms": 72.099}, "gstreamer_push": {"avg_ms": 0.859, "count": 37, "max_ms": 0.907, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.52, "count": 37, "max_ms": 63.572, "p95_ms": 51.753}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.684, "count": 37, "max_ms": 94.815, "p95_ms": 89.736}, "yolo11_640_fire_11class": {"avg_ms": 45.295, "count": 37, "max_ms": 65.041, "p95_ms": 62.906}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.429, "count": 37, "max_ms": 84.481, "p95_ms": 81.683}}, "truncated_count": 0}, "processing_total": {"avg_ms": 692.278, "count": 37, "max_ms": 782.806, "p95_ms": 754.564}, "read_interval": {"avg_ms": 702.675, "count": 36, "max_ms": 792.034, "p95_ms": 764.245}, "triton_wait": {"avg_ms": 69.274, "count": 37, "max_ms": 94.828, "p95_ms": 89.748}, "visualize": {"avg_ms": 11.903, "count": 37, "max_ms": 14.784, "p95_ms": 14.384}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.495, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 434, "tracking_summary": 11300}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10659.801, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557a4ec4db00] error while decoding MB 13 38, bytestream -35", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 408, "h264_decode_error_count": 1293, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:30:59+09:00", "last_error_summary": "[h264 @ 0x557a4ec4db00] error while decoding MB 13 38, bytestream -35", "last_exit_at": "2026-07-01T06:28:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 145.943, "last_started_at": "2026-07-01T06:28:33+09:00", "pid": 84008, "process": {"exists": false, "pid": 84008}, "restart_count": 445, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 446}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 875.61, "sample_count": 445, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356052, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:30:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.055, "last_frame_pushed_at": "2026-07-01T06:30:59+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-01T06:30:59+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-01T06:30:59.571435+09:00", "mtime_epoch": 1782855059.5714347, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192664}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:30:59+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5012.18, "name": "celery", "pid": 267, "rss_mb": 5012.18, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 51, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.716, "pushed": 1.716, "read": 1.716}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 55.762, "count": 50, "max_ms": 66.377, "p95_ms": 60.052}, "gstreamer_push": {"avg_ms": 0.865, "count": 51, "max_ms": 1.25, "p95_ms": 1.006}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.441, "count": 50, "max_ms": 64.313, "p95_ms": 59.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.97, "count": 50, "max_ms": 94.055, "p95_ms": 93.141}, "yolo11_640_fire_11class": {"avg_ms": 44.775, "count": 50, "max_ms": 61.121, "p95_ms": 60.939}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.136, "count": 50, "max_ms": 84.173, "p95_ms": 79.652}}, "truncated_count": 0}, "processing_total": {"avg_ms": 583.189, "count": 51, "max_ms": 649.343, "p95_ms": 637.632}, "read_interval": {"avg_ms": 592.443, "count": 51, "max_ms": 658.489, "p95_ms": 647.401}, "triton_wait": {"avg_ms": 74.384, "count": 50, "max_ms": 94.067, "p95_ms": 93.153}, "visualize": {"avg_ms": 11.002, "count": 50, "max_ms": 13.951, "p95_ms": 12.419}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.726, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 434, "tracking_summary": 11300}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10659.039, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:30:59+09:00", "last_error_summary": "[h264 @ 0x557a4ec4db00] error while decoding MB 13 38, bytestream -35", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 408, "h264_decode_error_count": 1293, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:30:59+09:00", "last_error_summary": "[h264 @ 0x557a4ec4db00] error while decoding MB 13 38, bytestream -35", "last_exit_at": "2026-07-01T06:31:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.469, "last_started_at": "2026-07-01T06:28:33+09:00", "pid": 84008, "process": {"exists": false, "pid": 84008}, "restart_count": 445, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 446}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 875.61, "sample_count": 445, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356053, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:30:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:31:00+09:00", "last_frame_read_age_sec": 0.575, "last_frame_read_at": "2026-07-01T06:30:59+09:00", "mjpeg": {"age_sec": 0.565, "exists": true, "mtime": "2026-07-01T06:30:59.571435+09:00", "mtime_epoch": 1782855059.5714347, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192664}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:31:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5012.18, "name": "celery", "pid": 267, "rss_mb": 5012.18, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.718, "pushed": 1.718, "read": 1.685}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 55.866, "count": 50, "max_ms": 66.377, "p95_ms": 60.052}, "gstreamer_push": {"avg_ms": 0.865, "count": 51, "max_ms": 1.25, "p95_ms": 1.006}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.704, "count": 50, "max_ms": 64.313, "p95_ms": 59.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.882, "count": 50, "max_ms": 94.055, "p95_ms": 93.141}, "yolo11_640_fire_11class": {"avg_ms": 45.113, "count": 50, "max_ms": 63.593, "p95_ms": 60.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.65, "count": 50, "max_ms": 84.173, "p95_ms": 80.945}}, "truncated_count": 0}, "processing_total": {"avg_ms": 583.833, "count": 51, "max_ms": 649.343, "p95_ms": 637.632}, "read_interval": {"avg_ms": 593.362, "count": 50, "max_ms": 658.489, "p95_ms": 647.401}, "triton_wait": {"avg_ms": 74.481, "count": 50, "max_ms": 94.067, "p95_ms": 93.153}, "visualize": {"avg_ms": 11.024, "count": 50, "max_ms": 13.951, "p95_ms": 12.419}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.679, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 434, "tracking_summary": 11300}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10659.031, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 408, "h264_decode_error_count": 1293, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:30:59+09:00", "last_error_summary": "[h264 @ 0x557a4ec4db00] error while decoding MB 13 38, bytestream -35", "last_exit_at": "2026-07-01T06:31:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.472, "last_started_at": "2026-07-01T06:28:33+09:00", "pid": 84008, "process": {"exists": false, "pid": 84008}, "restart_count": 445, "restart_count_per_hour": 22, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 446}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 875.61, "sample_count": 445, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356053, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:30:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T06:31:00+09:00", "last_frame_read_age_sec": 0.578, "last_frame_read_at": "2026-07-01T06:30:59+09:00", "mjpeg": {"age_sec": 0.568, "exists": true, "mtime": "2026-07-01T06:30:59.571435+09:00", "mtime_epoch": 1782855059.5714347, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192664}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:31:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5012.18, "name": "celery", "pid": 267, "rss_mb": 5012.18, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.718, "pushed": 1.718, "read": 1.685}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 55.866, "count": 50, "max_ms": 66.377, "p95_ms": 60.052}, "gstreamer_push": {"avg_ms": 0.865, "count": 51, "max_ms": 1.25, "p95_ms": 1.006}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.704, "count": 50, "max_ms": 64.313, "p95_ms": 59.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.882, "count": 50, "max_ms": 94.055, "p95_ms": 93.141}, "yolo11_640_fire_11class": {"avg_ms": 45.113, "count": 50, "max_ms": 63.593, "p95_ms": 60.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.65, "count": 50, "max_ms": 84.173, "p95_ms": 80.945}}, "truncated_count": 0}, "processing_total": {"avg_ms": 583.833, "count": 51, "max_ms": 649.343, "p95_ms": 637.632}, "read_interval": {"avg_ms": 593.362, "count": 50, "max_ms": 658.489, "p95_ms": 647.401}, "triton_wait": {"avg_ms": 74.481, "count": 50, "max_ms": 94.067, "p95_ms": 93.153}, "visualize": {"avg_ms": 11.024, "count": 50, "max_ms": 13.951, "p95_ms": 12.419}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.681, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 434, "tracking_summary": 11300}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10766.781, "limit_mb": 8796093022207.996}, "detail": {"pid": 84115, "restart_count": 446, "start_count": 447}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 408, "h264_decode_error_count": 1293, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:30:59+09:00", "last_error_summary": "[h264 @ 0x557a4ec4db00] error while decoding MB 13 38, bytestream -35", "last_exit_at": "2026-07-01T06:31:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:31:02+09:00", "pid": 84115, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.527, "name": "ffmpeg", "pid": 84115, "rss_mb": 138.527, "state": "S (sleeping)", "threads": 106}, "restart_count": 446, "restart_count_per_hour": 23, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 447}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 875.61, "sample_count": 438, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356053, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:30:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.231, "last_frame_pushed_at": "2026-07-01T06:31:00+09:00", "last_frame_read_age_sec": 4.803, "last_frame_read_at": "2026-07-01T06:30:59+09:00", "mjpeg": {"age_sec": 4.213, "exists": true, "mtime": "2026-07-01T06:31:00.151432+09:00", "mtime_epoch": 1782855060.1514323, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192918}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:31:04+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5012.18, "name": "celery", "pid": 267, "rss_mb": 5012.18, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.486, "pushed": 1.486, "read": 1.452}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 55.78, "count": 43, "max_ms": 61.741, "p95_ms": 59.494}, "gstreamer_push": {"avg_ms": 0.854, "count": 44, "max_ms": 1.006, "p95_ms": 0.89}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.922, "count": 43, "max_ms": 64.313, "p95_ms": 59.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.4, "count": 43, "max_ms": 94.055, "p95_ms": 93.141}, "yolo11_640_fire_11class": {"avg_ms": 45.271, "count": 43, "max_ms": 63.593, "p95_ms": 60.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.306, "count": 43, "max_ms": 84.173, "p95_ms": 80.945}}, "truncated_count": 0}, "processing_total": {"avg_ms": 582.191, "count": 44, "max_ms": 649.343, "p95_ms": 637.632}, "read_interval": {"avg_ms": 591.62, "count": 43, "max_ms": 658.489, "p95_ms": 647.401}, "triton_wait": {"avg_ms": 74.094, "count": 43, "max_ms": 94.067, "p95_ms": 93.153}, "visualize": {"avg_ms": 10.969, "count": 43, "max_ms": 12.511, "p95_ms": 12.227}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.609, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 436, "tracking_summary": 11302}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10667.508, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55eb557f6300] error while decoding MB 1 31, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 409, "h264_decode_error_count": 1294, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:33:09+09:00", "last_error_summary": "[h264 @ 0x55eb557f6300] error while decoding MB 1 31, bytestream -7", "last_exit_at": "2026-07-01T06:31:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.244, "last_started_at": "2026-07-01T06:31:02+09:00", "pid": 84115, "process": {"exists": false, "pid": 84115}, "restart_count": 446, "restart_count_per_hour": 23, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 447}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 678.368, "sample_count": 529, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356309, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:33:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.054, "last_frame_pushed_at": "2026-07-01T06:33:09+09:00", "last_frame_read_age_sec": 0.046, "last_frame_read_at": "2026-07-01T06:33:09+09:00", "mjpeg": {"age_sec": 0.036, "exists": true, "mtime": "2026-07-01T06:33:09.571160+09:00", "mtime_epoch": 1782855189.5711596, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191249}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:33:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5012.18, "name": "celery", "pid": 267, "rss_mb": 5012.18, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 78, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.628, "pushed": 2.628, "read": 2.628}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 43.099, "count": 78, "max_ms": 63.536, "p95_ms": 48.912}, "gstreamer_push": {"avg_ms": 0.875, "count": 78, "max_ms": 1.174, "p95_ms": 1.015}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.271, "count": 78, "max_ms": 41.674, "p95_ms": 35.888}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.932, "count": 78, "max_ms": 102.236, "p95_ms": 92.642}, "yolo11_640_fire_11class": {"avg_ms": 46.96, "count": 78, "max_ms": 71.426, "p95_ms": 62.192}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.737, "count": 78, "max_ms": 99.162, "p95_ms": 80.67}}, "truncated_count": 0}, "processing_total": {"avg_ms": 375.913, "count": 78, "max_ms": 443.51, "p95_ms": 435.681}, "read_interval": {"avg_ms": 385.495, "count": 78, "max_ms": 452.712, "p95_ms": 445.818}, "triton_wait": {"avg_ms": 73.945, "count": 78, "max_ms": 102.247, "p95_ms": 92.654}, "visualize": {"avg_ms": 11.024, "count": 78, "max_ms": 13.369, "p95_ms": 12.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.675, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 436, "tracking_summary": 11302}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10666.543, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:33:09+09:00", "last_error_summary": "[h264 @ 0x55eb557f6300] error while decoding MB 1 31, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 409, "h264_decode_error_count": 1294, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:33:09+09:00", "last_error_summary": "[h264 @ 0x55eb557f6300] error while decoding MB 1 31, bytestream -7", "last_exit_at": "2026-07-01T06:33:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.607, "last_started_at": "2026-07-01T06:31:02+09:00", "pid": 84115, "process": {"exists": false, "pid": 84115}, "restart_count": 446, "restart_count_per_hour": 23, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 447}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 678.023, "sample_count": 529, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356310, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:33:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T06:33:09+09:00", "last_frame_read_age_sec": 0.409, "last_frame_read_at": "2026-07-01T06:33:09+09:00", "mjpeg": {"age_sec": 0.399, "exists": true, "mtime": "2026-07-01T06:33:09.571160+09:00", "mtime_epoch": 1782855189.5711596, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191249}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:33:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5012.18, "name": "celery", "pid": 267, "rss_mb": 5012.18, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 77, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.634, "pushed": 2.634, "read": 2.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 43.148, "count": 78, "max_ms": 63.536, "p95_ms": 48.912}, "gstreamer_push": {"avg_ms": 0.871, "count": 78, "max_ms": 1.147, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.305, "count": 78, "max_ms": 41.674, "p95_ms": 35.888}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.971, "count": 78, "max_ms": 102.236, "p95_ms": 92.642}, "yolo11_640_fire_11class": {"avg_ms": 47.075, "count": 78, "max_ms": 71.426, "p95_ms": 62.192}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.604, "count": 78, "max_ms": 99.162, "p95_ms": 80.67}}, "truncated_count": 0}, "processing_total": {"avg_ms": 375.45, "count": 78, "max_ms": 443.51, "p95_ms": 434.358}, "read_interval": {"avg_ms": 384.64, "count": 77, "max_ms": 452.712, "p95_ms": 443.989}, "triton_wait": {"avg_ms": 73.984, "count": 78, "max_ms": 102.247, "p95_ms": 92.654}, "visualize": {"avg_ms": 11.004, "count": 78, "max_ms": 13.369, "p95_ms": 12.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.608, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 436, "tracking_summary": 11302}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10666.527, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 409, "h264_decode_error_count": 1294, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:33:09+09:00", "last_error_summary": "[h264 @ 0x55eb557f6300] error while decoding MB 1 31, bytestream -7", "last_exit_at": "2026-07-01T06:33:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.611, "last_started_at": "2026-07-01T06:31:02+09:00", "pid": 84115, "process": {"exists": false, "pid": 84115}, "restart_count": 446, "restart_count_per_hour": 23, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 447}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 678.023, "sample_count": 529, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356310, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:33:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T06:33:09+09:00", "last_frame_read_age_sec": 0.412, "last_frame_read_at": "2026-07-01T06:33:09+09:00", "mjpeg": {"age_sec": 0.403, "exists": true, "mtime": "2026-07-01T06:33:09.571160+09:00", "mtime_epoch": 1782855189.5711596, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191249}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:33:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5012.18, "name": "celery", "pid": 267, "rss_mb": 5012.18, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 77, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.634, "pushed": 2.634, "read": 2.6}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 43.148, "count": 78, "max_ms": 63.536, "p95_ms": 48.912}, "gstreamer_push": {"avg_ms": 0.871, "count": 78, "max_ms": 1.147, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.305, "count": 78, "max_ms": 41.674, "p95_ms": 35.888}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.971, "count": 78, "max_ms": 102.236, "p95_ms": 92.642}, "yolo11_640_fire_11class": {"avg_ms": 47.075, "count": 78, "max_ms": 71.426, "p95_ms": 62.192}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.604, "count": 78, "max_ms": 99.162, "p95_ms": 80.67}}, "truncated_count": 0}, "processing_total": {"avg_ms": 375.45, "count": 78, "max_ms": 443.51, "p95_ms": 434.358}, "read_interval": {"avg_ms": 384.64, "count": 77, "max_ms": 452.712, "p95_ms": 443.989}, "triton_wait": {"avg_ms": 73.984, "count": 78, "max_ms": 102.247, "p95_ms": 92.654}, "visualize": {"avg_ms": 11.004, "count": 78, "max_ms": 13.369, "p95_ms": 12.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.611, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 436, "tracking_summary": 11302}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10775.145, "limit_mb": 8796093022207.996}, "detail": {"pid": 84282, "restart_count": 447, "start_count": 448}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 409, "h264_decode_error_count": 1294, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:33:09+09:00", "last_error_summary": "[h264 @ 0x55eb557f6300] error while decoding MB 1 31, bytestream -7", "last_exit_at": "2026-07-01T06:33:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:33:12+09:00", "pid": 84282, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.855, "name": "ffmpeg", "pid": 84282, "rss_mb": 138.855, "state": "S (sleeping)", "threads": 106}, "restart_count": 447, "restart_count_per_hour": 24, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 448}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 674.797, "sample_count": 523, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356310, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:33:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.594, "last_frame_pushed_at": "2026-07-01T06:33:09+09:00", "last_frame_read_age_sec": 4.999, "last_frame_read_at": "2026-07-01T06:33:09+09:00", "mjpeg": {"age_sec": 4.577, "exists": true, "mtime": "2026-07-01T06:33:09.983160+09:00", "mtime_epoch": 1782855189.9831595, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187306}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:33:14+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5012.18, "name": "celery", "pid": 267, "rss_mb": 5012.18, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 66, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.2, "pushed": 2.2, "read": 2.2}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 42.907, "count": 66, "max_ms": 50.63, "p95_ms": 48.09}, "gstreamer_push": {"avg_ms": 0.872, "count": 66, "max_ms": 1.147, "p95_ms": 0.912}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.381, "count": 66, "max_ms": 41.674, "p95_ms": 35.338}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.808, "count": 66, "max_ms": 102.236, "p95_ms": 92.642}, "yolo11_640_fire_11class": {"avg_ms": 47.241, "count": 66, "max_ms": 71.426, "p95_ms": 62.192}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.687, "count": 66, "max_ms": 99.162, "p95_ms": 79.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 375.387, "count": 66, "max_ms": 443.51, "p95_ms": 434.358}, "read_interval": {"avg_ms": 384.402, "count": 66, "max_ms": 452.712, "p95_ms": 443.626}, "triton_wait": {"avg_ms": 73.821, "count": 66, "max_ms": 102.247, "p95_ms": 92.654}, "visualize": {"avg_ms": 10.937, "count": 66, "max_ms": 13.369, "p95_ms": 12.707}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 436, "tracking_summary": 11305}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10669.473, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56329f3fc440] error while decoding MB 33 28, bytestream -27", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 410, "h264_decode_error_count": 1295, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:35:14+09:00", "last_error_summary": "[h264 @ 0x56329f3fc440] error while decoding MB 33 28, bytestream -27", "last_exit_at": "2026-07-01T06:33:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.605, "last_started_at": "2026-07-01T06:33:12+09:00", "pid": 84282, "process": {"exists": false, "pid": 84282}, "restart_count": 447, "restart_count_per_hour": 23, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 448}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 623.941, "sample_count": 700, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356674, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:35:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T06:35:14+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T06:35:14+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T06:35:14.127331+09:00", "mtime_epoch": 1782855314.127331, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194780}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:35:14+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5012.438, "name": "celery", "pid": 267, "rss_mb": 5012.438, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 91, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.062, "pushed": 3.062, "read": 3.062}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 39.721, "count": 91, "max_ms": 44.641, "p95_ms": 42.969}, "gstreamer_push": {"avg_ms": 0.868, "count": 91, "max_ms": 1.237, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.042, "count": 91, "max_ms": 48.193, "p95_ms": 40.65}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.401, "count": 91, "max_ms": 92.892, "p95_ms": 87.807}, "yolo11_640_fire_11class": {"avg_ms": 46.63, "count": 91, "max_ms": 64.081, "p95_ms": 57.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.935, "count": 91, "max_ms": 85.411, "p95_ms": 76.986}}, "truncated_count": 0}, "processing_total": {"avg_ms": 319.8, "count": 91, "max_ms": 393.124, "p95_ms": 366.655}, "read_interval": {"avg_ms": 329.328, "count": 91, "max_ms": 404.006, "p95_ms": 375.821}, "triton_wait": {"avg_ms": 74.414, "count": 91, "max_ms": 92.903, "p95_ms": 87.818}, "visualize": {"avg_ms": 9.516, "count": 91, "max_ms": 12.228, "p95_ms": 11.687}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.72, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 436, "tracking_summary": 11305}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10668.688, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:35:14+09:00", "last_error_summary": "[h264 @ 0x56329f3fc440] error while decoding MB 33 28, bytestream -27", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 410, "h264_decode_error_count": 1295, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:35:14+09:00", "last_error_summary": "[h264 @ 0x56329f3fc440] error while decoding MB 33 28, bytestream -27", "last_exit_at": "2026-07-01T06:35:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.892, "last_started_at": "2026-07-01T06:33:12+09:00", "pid": 84282, "process": {"exists": false, "pid": 84282}, "restart_count": 447, "restart_count_per_hour": 23, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 448}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 623.941, "sample_count": 700, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:35:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T06:35:14+09:00", "last_frame_read_age_sec": 0.337, "last_frame_read_at": "2026-07-01T06:35:14+09:00", "mjpeg": {"age_sec": 0.324, "exists": true, "mtime": "2026-07-01T06:35:14.127331+09:00", "mtime_epoch": 1782855314.127331, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194780}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:35:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5012.438, "name": "celery", "pid": 267, "rss_mb": 5012.438, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 91, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.034, "pushed": 3.034, "read": 3.034}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 39.787, "count": 91, "max_ms": 44.963, "p95_ms": 43.014}, "gstreamer_push": {"avg_ms": 0.868, "count": 91, "max_ms": 1.237, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.894, "count": 91, "max_ms": 48.193, "p95_ms": 40.323}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.391, "count": 91, "max_ms": 92.892, "p95_ms": 87.807}, "yolo11_640_fire_11class": {"avg_ms": 46.614, "count": 91, "max_ms": 64.081, "p95_ms": 57.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.96, "count": 91, "max_ms": 85.411, "p95_ms": 76.986}}, "truncated_count": 0}, "processing_total": {"avg_ms": 320.155, "count": 91, "max_ms": 393.124, "p95_ms": 366.655}, "read_interval": {"avg_ms": 329.328, "count": 91, "max_ms": 404.006, "p95_ms": 375.821}, "triton_wait": {"avg_ms": 74.404, "count": 91, "max_ms": 92.903, "p95_ms": 87.818}, "visualize": {"avg_ms": 9.507, "count": 91, "max_ms": 12.228, "p95_ms": 11.687}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 436, "tracking_summary": 11305}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10668.68, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 410, "h264_decode_error_count": 1295, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:35:14+09:00", "last_error_summary": "[h264 @ 0x56329f3fc440] error while decoding MB 33 28, bytestream -27", "last_exit_at": "2026-07-01T06:35:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.897, "last_started_at": "2026-07-01T06:33:12+09:00", "pid": 84282, "process": {"exists": false, "pid": 84282}, "restart_count": 447, "restart_count_per_hour": 23, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 448}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 623.941, "sample_count": 700, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:35:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T06:35:14+09:00", "last_frame_read_age_sec": 0.342, "last_frame_read_at": "2026-07-01T06:35:14+09:00", "mjpeg": {"age_sec": 0.329, "exists": true, "mtime": "2026-07-01T06:35:14.127331+09:00", "mtime_epoch": 1782855314.127331, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194780}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:35:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5012.438, "name": "celery", "pid": 267, "rss_mb": 5012.438, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 90, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.066, "pushed": 3.066, "read": 3.032}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 39.787, "count": 91, "max_ms": 44.963, "p95_ms": 43.014}, "gstreamer_push": {"avg_ms": 0.868, "count": 91, "max_ms": 1.237, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.894, "count": 91, "max_ms": 48.193, "p95_ms": 40.323}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.391, "count": 91, "max_ms": 92.892, "p95_ms": 87.807}, "yolo11_640_fire_11class": {"avg_ms": 46.614, "count": 91, "max_ms": 64.081, "p95_ms": 57.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.96, "count": 91, "max_ms": 85.411, "p95_ms": 76.986}}, "truncated_count": 0}, "processing_total": {"avg_ms": 320.155, "count": 91, "max_ms": 393.124, "p95_ms": 366.655}, "read_interval": {"avg_ms": 329.563, "count": 90, "max_ms": 404.006, "p95_ms": 375.821}, "triton_wait": {"avg_ms": 74.404, "count": 91, "max_ms": 92.903, "p95_ms": 87.818}, "visualize": {"avg_ms": 9.507, "count": 91, "max_ms": 12.228, "p95_ms": 11.687}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.685, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 436, "tracking_summary": 11305}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10776.98, "limit_mb": 8796093022207.996}, "detail": {"pid": 84389, "restart_count": 448, "start_count": 449}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 410, "h264_decode_error_count": 1295, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:35:14+09:00", "last_error_summary": "[h264 @ 0x56329f3fc440] error while decoding MB 33 28, bytestream -27", "last_exit_at": "2026-07-01T06:35:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:35:16+09:00", "pid": 84389, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.809, "name": "ffmpeg", "pid": 84389, "rss_mb": 138.809, "state": "S (sleeping)", "threads": 106}, "restart_count": 448, "restart_count_per_hour": 24, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 449}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 623.941, "sample_count": 693, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1356675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:35:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.109, "last_frame_pushed_at": "2026-07-01T06:35:14+09:00", "last_frame_read_age_sec": 4.441, "last_frame_read_at": "2026-07-01T06:35:14+09:00", "mjpeg": {"age_sec": 4.092, "exists": true, "mtime": "2026-07-01T06:35:14.463332+09:00", "mtime_epoch": 1782855314.463332, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190542}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:35:18+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5012.438, "name": "celery", "pid": 267, "rss_mb": 5012.438, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 78, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.6, "pushed": 2.6, "read": 2.6}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 39.786, "count": 78, "max_ms": 44.963, "p95_ms": 43.197}, "gstreamer_push": {"avg_ms": 0.868, "count": 78, "max_ms": 1.237, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.638, "count": 78, "max_ms": 41.342, "p95_ms": 38.414}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.283, "count": 78, "max_ms": 92.892, "p95_ms": 87.807}, "yolo11_640_fire_11class": {"avg_ms": 46.859, "count": 78, "max_ms": 64.081, "p95_ms": 59.405}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.454, "count": 78, "max_ms": 85.411, "p95_ms": 81.596}}, "truncated_count": 0}, "processing_total": {"avg_ms": 322.403, "count": 78, "max_ms": 393.124, "p95_ms": 367.424}, "read_interval": {"avg_ms": 332.289, "count": 78, "max_ms": 404.006, "p95_ms": 377.711}, "triton_wait": {"avg_ms": 74.296, "count": 78, "max_ms": 92.903, "p95_ms": 87.818}, "visualize": {"avg_ms": 9.635, "count": 78, "max_ms": 12.228, "p95_ms": 11.964}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11309}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10677.008, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562a98285880] error while decoding MB 60 26, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 411, "h264_decode_error_count": 1296, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:37:17+09:00", "last_error_summary": "[h264 @ 0x562a98285880] error while decoding MB 60 26, bytestream -15", "last_exit_at": "2026-07-01T06:35:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.321, "last_started_at": "2026-07-01T06:35:16+09:00", "pid": 84389, "process": {"exists": false, "pid": 84389}, "restart_count": 448, "restart_count_per_hour": 24, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 449}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 405.183, "sample_count": 909, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1357089, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:37:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T06:37:17+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T06:37:17+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T06:37:17.835830+09:00", "mtime_epoch": 1782855437.8358305, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:37:17+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5012.695, "name": "celery", "pid": 267, "rss_mb": 5012.695, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 91, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.069, "pushed": 3.069, "read": 3.069}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 40.392, "count": 91, "max_ms": 49.81, "p95_ms": 44.844}, "gstreamer_push": {"avg_ms": 0.867, "count": 91, "max_ms": 1.365, "p95_ms": 0.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.821, "count": 91, "max_ms": 53.95, "p95_ms": 47.136}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.201, "count": 91, "max_ms": 104.479, "p95_ms": 94.524}, "yolo11_640_fire_11class": {"avg_ms": 46.765, "count": 91, "max_ms": 66.246, "p95_ms": 61.276}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.733, "count": 91, "max_ms": 90.597, "p95_ms": 85.681}}, "truncated_count": 0}, "processing_total": {"avg_ms": 319.862, "count": 91, "max_ms": 390.645, "p95_ms": 379.626}, "read_interval": {"avg_ms": 329.473, "count": 91, "max_ms": 399.833, "p95_ms": 388.979}, "triton_wait": {"avg_ms": 72.214, "count": 91, "max_ms": 104.491, "p95_ms": 94.537}, "visualize": {"avg_ms": 11.255, "count": 91, "max_ms": 17.165, "p95_ms": 15.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.652, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11309}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10676.176, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:37:17+09:00", "last_error_summary": "[h264 @ 0x562a98285880] error while decoding MB 60 26, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 411, "h264_decode_error_count": 1296, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:37:17+09:00", "last_error_summary": "[h264 @ 0x562a98285880] error while decoding MB 60 26, bytestream -15", "last_exit_at": "2026-07-01T06:37:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.594, "last_started_at": "2026-07-01T06:35:16+09:00", "pid": 84389, "process": {"exists": false, "pid": 84389}, "restart_count": 448, "restart_count_per_hour": 24, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 449}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 405.183, "sample_count": 910, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1357090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:37:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:37:18+09:00", "last_frame_read_age_sec": 0.322, "last_frame_read_at": "2026-07-01T06:37:17+09:00", "mjpeg": {"age_sec": 0.312, "exists": true, "mtime": "2026-07-01T06:37:17.835830+09:00", "mtime_epoch": 1782855437.8358305, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:37:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5012.695, "name": "celery", "pid": 267, "rss_mb": 5012.695, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 91, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.074, "pushed": 3.074, "read": 3.041}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 40.377, "count": 91, "max_ms": 49.81, "p95_ms": 44.844}, "gstreamer_push": {"avg_ms": 0.868, "count": 92, "max_ms": 1.365, "p95_ms": 0.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.831, "count": 91, "max_ms": 53.95, "p95_ms": 47.136}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.923, "count": 91, "max_ms": 103.753, "p95_ms": 91.875}, "yolo11_640_fire_11class": {"avg_ms": 46.543, "count": 91, "max_ms": 66.246, "p95_ms": 61.243}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.393, "count": 91, "max_ms": 87.659, "p95_ms": 83.964}}, "truncated_count": 0}, "processing_total": {"avg_ms": 319.837, "count": 92, "max_ms": 390.645, "p95_ms": 379.626}, "read_interval": {"avg_ms": 329.473, "count": 91, "max_ms": 399.833, "p95_ms": 388.979}, "triton_wait": {"avg_ms": 71.936, "count": 91, "max_ms": 103.766, "p95_ms": 91.887}, "visualize": {"avg_ms": 11.165, "count": 91, "max_ms": 16.702, "p95_ms": 15.409}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11309}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10676.215, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 411, "h264_decode_error_count": 1296, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:37:17+09:00", "last_error_summary": "[h264 @ 0x562a98285880] error while decoding MB 60 26, bytestream -15", "last_exit_at": "2026-07-01T06:37:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.597, "last_started_at": "2026-07-01T06:35:16+09:00", "pid": 84389, "process": {"exists": false, "pid": 84389}, "restart_count": 448, "restart_count_per_hour": 24, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 449}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 405.183, "sample_count": 910, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1357090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:37:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T06:37:18+09:00", "last_frame_read_age_sec": 0.325, "last_frame_read_at": "2026-07-01T06:37:17+09:00", "mjpeg": {"age_sec": 0.316, "exists": true, "mtime": "2026-07-01T06:37:17.835830+09:00", "mtime_epoch": 1782855437.8358305, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:37:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5012.695, "name": "celery", "pid": 267, "rss_mb": 5012.695, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 91, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.074, "pushed": 3.074, "read": 3.041}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 40.377, "count": 91, "max_ms": 49.81, "p95_ms": 44.844}, "gstreamer_push": {"avg_ms": 0.868, "count": 92, "max_ms": 1.365, "p95_ms": 0.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.831, "count": 91, "max_ms": 53.95, "p95_ms": 47.136}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.923, "count": 91, "max_ms": 103.753, "p95_ms": 91.875}, "yolo11_640_fire_11class": {"avg_ms": 46.543, "count": 91, "max_ms": 66.246, "p95_ms": 61.243}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.393, "count": 91, "max_ms": 87.659, "p95_ms": 83.964}}, "truncated_count": 0}, "processing_total": {"avg_ms": 319.837, "count": 92, "max_ms": 390.645, "p95_ms": 379.626}, "read_interval": {"avg_ms": 329.473, "count": 91, "max_ms": 399.833, "p95_ms": 388.979}, "triton_wait": {"avg_ms": 71.936, "count": 91, "max_ms": 103.766, "p95_ms": 91.887}, "visualize": {"avg_ms": 11.165, "count": 91, "max_ms": 16.702, "p95_ms": 15.409}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11309}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10779.391, "limit_mb": 8796093022207.996}, "detail": {"pid": 84556, "restart_count": 449, "start_count": 450}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 411, "h264_decode_error_count": 1296, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:37:17+09:00", "last_error_summary": "[h264 @ 0x562a98285880] error while decoding MB 60 26, bytestream -15", "last_exit_at": "2026-07-01T06:37:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:37:20+09:00", "pid": 84556, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.277, "name": "ffmpeg", "pid": 84556, "rss_mb": 133.277, "state": "S (sleeping)", "threads": 106}, "restart_count": 449, "restart_count_per_hour": 25, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 450}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 400.942, "sample_count": 901, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1357090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:37:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.228, "last_frame_pushed_at": "2026-07-01T06:37:18+09:00", "last_frame_read_age_sec": 4.547, "last_frame_read_at": "2026-07-01T06:37:17+09:00", "mjpeg": {"age_sec": 4.209, "exists": true, "mtime": "2026-07-01T06:37:18.163832+09:00", "mtime_epoch": 1782855438.1638322, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189741}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:37:22+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5012.695, "name": "celery", "pid": 267, "rss_mb": 5012.695, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 79, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.675, "pushed": 2.675, "read": 2.641}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 40.342, "count": 79, "max_ms": 49.81, "p95_ms": 44.862}, "gstreamer_push": {"avg_ms": 0.867, "count": 80, "max_ms": 1.365, "p95_ms": 0.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.107, "count": 79, "max_ms": 53.95, "p95_ms": 49.446}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.316, "count": 79, "max_ms": 101.926, "p95_ms": 90.033}, "yolo11_640_fire_11class": {"avg_ms": 46.198, "count": 79, "max_ms": 66.246, "p95_ms": 59.227}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.915, "count": 79, "max_ms": 87.659, "p95_ms": 83.964}}, "truncated_count": 0}, "processing_total": {"avg_ms": 316.03, "count": 80, "max_ms": 390.645, "p95_ms": 379.6}, "read_interval": {"avg_ms": 325.631, "count": 79, "max_ms": 399.833, "p95_ms": 391.316}, "triton_wait": {"avg_ms": 71.329, "count": 79, "max_ms": 101.939, "p95_ms": 90.045}, "visualize": {"avg_ms": 10.903, "count": 79, "max_ms": 16.702, "p95_ms": 15.409}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11312}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10678.023, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:37:17+09:00", "last_error_summary": "[h264 @ 0x562a98285880] error while decoding MB 60 26, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 411, "h264_decode_error_count": 1296, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:37:17+09:00", "last_error_summary": "[h264 @ 0x562a98285880] error while decoding MB 60 26, bytestream -15", "last_exit_at": "2026-07-01T06:39:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 117.672, "last_started_at": "2026-07-01T06:37:20+09:00", "pid": 84556, "process": {"exists": false, "pid": 84556}, "restart_count": 449, "restart_count_per_hour": 24, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 450}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 359.96, "sample_count": 997, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1357504, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:39:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T06:39:18+09:00", "last_frame_read_age_sec": 0.29, "last_frame_read_at": "2026-07-01T06:39:17+09:00", "mjpeg": {"age_sec": 0.279, "exists": true, "mtime": "2026-07-01T06:39:17.764561+09:00", "mtime_epoch": 1782855557.7645612, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191909}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:39:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5012.695, "name": "celery", "pid": 267, "rss_mb": 5012.695, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.018, "pushed": 4.018, "read": 3.985}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.825, "count": 120, "max_ms": 42.549, "p95_ms": 41.258}, "gstreamer_push": {"avg_ms": 0.877, "count": 120, "max_ms": 1.312, "p95_ms": 0.977}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.347, "count": 120, "max_ms": 43.961, "p95_ms": 39.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.428, "count": 120, "max_ms": 98.69, "p95_ms": 94.037}, "yolo11_640_fire_11class": {"avg_ms": 46.16, "count": 120, "max_ms": 66.444, "p95_ms": 62.345}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.84, "count": 120, "max_ms": 85.944, "p95_ms": 82.143}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.147, "count": 120, "max_ms": 338.211, "p95_ms": 313.479}, "read_interval": {"avg_ms": 250.273, "count": 119, "max_ms": 347.447, "p95_ms": 327.749}, "triton_wait": {"avg_ms": 71.591, "count": 120, "max_ms": 98.702, "p95_ms": 94.05}, "visualize": {"avg_ms": 7.225, "count": 120, "max_ms": 10.53, "p95_ms": 9.449}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.864, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11312}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10678.016, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 411, "h264_decode_error_count": 1296, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:37:17+09:00", "last_error_summary": "[h264 @ 0x562a98285880] error while decoding MB 60 26, bytestream -15", "last_exit_at": "2026-07-01T06:39:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 117.675, "last_started_at": "2026-07-01T06:37:20+09:00", "pid": 84556, "process": {"exists": false, "pid": 84556}, "restart_count": 449, "restart_count_per_hour": 24, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 450}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 359.96, "sample_count": 997, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1357504, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:39:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T06:39:18+09:00", "last_frame_read_age_sec": 0.293, "last_frame_read_at": "2026-07-01T06:39:17+09:00", "mjpeg": {"age_sec": 0.283, "exists": true, "mtime": "2026-07-01T06:39:17.764561+09:00", "mtime_epoch": 1782855557.7645612, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191909}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:39:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5012.695, "name": "celery", "pid": 267, "rss_mb": 5012.695, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.018, "pushed": 4.018, "read": 3.984}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.825, "count": 120, "max_ms": 42.549, "p95_ms": 41.258}, "gstreamer_push": {"avg_ms": 0.877, "count": 120, "max_ms": 1.312, "p95_ms": 0.977}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.347, "count": 120, "max_ms": 43.961, "p95_ms": 39.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.428, "count": 120, "max_ms": 98.69, "p95_ms": 94.037}, "yolo11_640_fire_11class": {"avg_ms": 46.16, "count": 120, "max_ms": 66.444, "p95_ms": 62.345}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.84, "count": 120, "max_ms": 85.944, "p95_ms": 82.143}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.147, "count": 120, "max_ms": 338.211, "p95_ms": 313.479}, "read_interval": {"avg_ms": 250.273, "count": 119, "max_ms": 347.447, "p95_ms": 327.749}, "triton_wait": {"avg_ms": 71.591, "count": 120, "max_ms": 98.702, "p95_ms": 94.05}, "visualize": {"avg_ms": 7.225, "count": 120, "max_ms": 10.53, "p95_ms": 9.449}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11312}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10786.449, "limit_mb": 8796093022207.996}, "detail": {"pid": 84663, "restart_count": 450, "start_count": 451}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 411, "h264_decode_error_count": 1296, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:37:17+09:00", "last_error_summary": "[h264 @ 0x562a98285880] error while decoding MB 60 26, bytestream -15", "last_exit_at": "2026-07-01T06:39:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:39:20+09:00", "pid": 84663, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.422, "name": "ffmpeg", "pid": 84663, "rss_mb": 139.422, "state": "S (sleeping)", "threads": 106}, "restart_count": 450, "restart_count_per_hour": 25, "rtsp_timeout_count": 386, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 451}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 361.17, "sample_count": 981, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1357504, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:39:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.885, "last_frame_pushed_at": "2026-07-01T06:39:18+09:00", "last_frame_read_age_sec": 5.17, "last_frame_read_at": "2026-07-01T06:39:17+09:00", "mjpeg": {"age_sec": 4.868, "exists": true, "mtime": "2026-07-01T06:39:18.056563+09:00", "mtime_epoch": 1782855558.0565631, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:39:22+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5012.695, "name": "celery", "pid": 267, "rss_mb": 5012.695, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.186, "pushed": 3.186, "read": 3.153}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 37.214, "count": 95, "max_ms": 42.549, "p95_ms": 41.258}, "gstreamer_push": {"avg_ms": 0.872, "count": 95, "max_ms": 1.312, "p95_ms": 0.946}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.472, "count": 95, "max_ms": 43.131, "p95_ms": 37.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.591, "count": 95, "max_ms": 98.69, "p95_ms": 94.19}, "yolo11_640_fire_11class": {"avg_ms": 47.81, "count": 95, "max_ms": 66.444, "p95_ms": 62.39}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.903, "count": 95, "max_ms": 85.944, "p95_ms": 83.407}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.176, "count": 95, "max_ms": 338.211, "p95_ms": 317.946}, "read_interval": {"avg_ms": 264.293, "count": 94, "max_ms": 347.447, "p95_ms": 327.999}, "triton_wait": {"avg_ms": 73.681, "count": 95, "max_ms": 98.702, "p95_ms": 94.202}, "visualize": {"avg_ms": 7.527, "count": 95, "max_ms": 10.53, "p95_ms": 9.752}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.816, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11319}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10679.574, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5575bba34540] cabac decode of qscale diff failed at 71 28 [h264 @ 0x5575bba34540] error while decoding MB 71 28, bytestream 138797 [h264 @ 0x5575bba50480] error while decoding MB 68 41, bytestream -8 [in#0/rtsp @ 0x5575bb90ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5575bb90ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 412, "h264_decode_error_count": 1299, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:42:04+09:00", "last_error_summary": "[h264 @ 0x5575bba34540] cabac decode of qscale diff failed at 71 28 [h264 @ 0x5575bba34540] error while decoding MB 71 28, bytestream 138797 [h264 @ 0x5575bba50480] error while decoding MB 68 41, bytestream -8 [in#0/rtsp @ 0x5575bb90ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5575bb90ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:39:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.013, "last_started_at": "2026-07-01T06:39:20+09:00", "pid": 84663, "process": {"exists": false, "pid": 84663}, "restart_count": 450, "restart_count_per_hour": 24, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 451}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 316.138, "sample_count": 1307, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1358352, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:42:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-01T06:42:04+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T06:42:04+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T06:42:04.889889+09:00", "mtime_epoch": 1782855724.8898895, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189234}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:42:04+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5013.211, "name": "celery", "pid": 267, "rss_mb": 5013.211, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.134, "pushed": 2.134, "read": 2.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.557, "count": 63, "max_ms": 45.18, "p95_ms": 38.136}, "gstreamer_push": {"avg_ms": 1.04, "count": 64, "max_ms": 1.239, "p95_ms": 1.149}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.043, "count": 63, "max_ms": 37.167, "p95_ms": 36.207}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.199, "count": 63, "max_ms": 92.003, "p95_ms": 88.475}, "yolo11_640_fire_11class": {"avg_ms": 48.526, "count": 63, "max_ms": 63.659, "p95_ms": 62.24}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.007, "count": 63, "max_ms": 81.001, "p95_ms": 76.883}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.399, "count": 64, "max_ms": 194.099, "p95_ms": 181.49}, "read_interval": {"avg_ms": 470.001, "count": 64, "max_ms": 10162.963, "p95_ms": 197.365}, "triton_wait": {"avg_ms": 68.212, "count": 63, "max_ms": 92.014, "p95_ms": 88.487}, "visualize": {"avg_ms": 4.609, "count": 63, "max_ms": 6.287, "p95_ms": 5.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11319}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10678.988, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:42:04+09:00", "last_error_summary": "[h264 @ 0x5575bba34540] cabac decode of qscale diff failed at 71 28 [h264 @ 0x5575bba34540] error while decoding MB 71 28, bytestream 138797 [h264 @ 0x5575bba50480] error while decoding MB 68 41, bytestream -8 [in#0/rtsp @ 0x5575bb90ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5575bb90ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 412, "h264_decode_error_count": 1299, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:42:04+09:00", "last_error_summary": "[h264 @ 0x5575bba34540] cabac decode of qscale diff failed at 71 28 [h264 @ 0x5575bba34540] error while decoding MB 71 28, bytestream 138797 [h264 @ 0x5575bba50480] error while decoding MB 68 41, bytestream -8 [in#0/rtsp @ 0x5575bb90ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5575bb90ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:42:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.074, "last_started_at": "2026-07-01T06:39:20+09:00", "pid": 84663, "process": {"exists": false, "pid": 84663}, "restart_count": 450, "restart_count_per_hour": 24, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 451}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 316.138, "sample_count": 1308, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1358353, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:42:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T06:42:04+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-01T06:42:04+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T06:42:04.889889+09:00", "mtime_epoch": 1782855724.8898895, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189234}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:42:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5013.211, "name": "celery", "pid": 267, "rss_mb": 5013.211, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.14, "pushed": 2.14, "read": 2.107}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.47, "count": 64, "max_ms": 45.18, "p95_ms": 38.136}, "gstreamer_push": {"avg_ms": 1.042, "count": 64, "max_ms": 1.239, "p95_ms": 1.155}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.028, "count": 64, "max_ms": 37.167, "p95_ms": 36.207}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.306, "count": 64, "max_ms": 92.003, "p95_ms": 88.475}, "yolo11_640_fire_11class": {"avg_ms": 48.443, "count": 64, "max_ms": 63.659, "p95_ms": 62.24}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.05, "count": 64, "max_ms": 81.001, "p95_ms": 76.883}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.935, "count": 64, "max_ms": 194.099, "p95_ms": 181.49}, "read_interval": {"avg_ms": 475.075, "count": 63, "max_ms": 10162.963, "p95_ms": 197.365}, "triton_wait": {"avg_ms": 68.319, "count": 64, "max_ms": 92.014, "p95_ms": 88.487}, "visualize": {"avg_ms": 4.587, "count": 64, "max_ms": 6.287, "p95_ms": 5.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11319}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10678.934, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 412, "h264_decode_error_count": 1299, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:42:04+09:00", "last_error_summary": "[h264 @ 0x5575bba34540] cabac decode of qscale diff failed at 71 28 [h264 @ 0x5575bba34540] error while decoding MB 71 28, bytestream 138797 [h264 @ 0x5575bba50480] error while decoding MB 68 41, bytestream -8 [in#0/rtsp @ 0x5575bb90ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5575bb90ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:42:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.079, "last_started_at": "2026-07-01T06:39:20+09:00", "pid": 84663, "process": {"exists": false, "pid": 84663}, "restart_count": 450, "restart_count_per_hour": 24, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 451}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 316.138, "sample_count": 1308, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1358353, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:42:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T06:42:04+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-01T06:42:04+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-01T06:42:04.889889+09:00", "mtime_epoch": 1782855724.8898895, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189234}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:42:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5013.211, "name": "celery", "pid": 267, "rss_mb": 5013.211, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.14, "pushed": 2.14, "read": 2.106}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.47, "count": 64, "max_ms": 45.18, "p95_ms": 38.136}, "gstreamer_push": {"avg_ms": 1.042, "count": 64, "max_ms": 1.239, "p95_ms": 1.155}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.028, "count": 64, "max_ms": 37.167, "p95_ms": 36.207}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.306, "count": 64, "max_ms": 92.003, "p95_ms": 88.475}, "yolo11_640_fire_11class": {"avg_ms": 48.443, "count": 64, "max_ms": 63.659, "p95_ms": 62.24}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.05, "count": 64, "max_ms": 81.001, "p95_ms": 76.883}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.935, "count": 64, "max_ms": 194.099, "p95_ms": 181.49}, "read_interval": {"avg_ms": 475.075, "count": 63, "max_ms": 10162.963, "p95_ms": 197.365}, "triton_wait": {"avg_ms": 68.319, "count": 64, "max_ms": 92.014, "p95_ms": 88.487}, "visualize": {"avg_ms": 4.587, "count": 64, "max_ms": 6.287, "p95_ms": 5.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11319}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10792.566, "limit_mb": 8796093022207.996}, "detail": {"pid": 84770, "restart_count": 451, "start_count": 452}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 412, "h264_decode_error_count": 1299, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:42:04+09:00", "last_error_summary": "[h264 @ 0x5575bba34540] cabac decode of qscale diff failed at 71 28 [h264 @ 0x5575bba34540] error while decoding MB 71 28, bytestream 138797 [h264 @ 0x5575bba50480] error while decoding MB 68 41, bytestream -8 [in#0/rtsp @ 0x5575bb90ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5575bb90ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:42:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:42:07+09:00", "pid": 84770, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.594, "name": "ffmpeg", "pid": 84770, "rss_mb": 144.594, "state": "S (sleeping)", "threads": 106}, "restart_count": 451, "restart_count_per_hour": 25, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 452}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 313.938, "sample_count": 1294, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1358353, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:42:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.208, "last_frame_pushed_at": "2026-07-01T06:42:04+09:00", "last_frame_read_age_sec": 4.319, "last_frame_read_at": "2026-07-01T06:42:04+09:00", "mjpeg": {"age_sec": 4.191, "exists": true, "mtime": "2026-07-01T06:42:05.009891+09:00", "mtime_epoch": 1782855725.0098906, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187813}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:42:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5013.211, "name": "celery", "pid": 267, "rss_mb": 5013.211, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 38, "pushed_frames": 38, "read_frames": 37, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.269, "pushed": 1.269, "read": 1.236}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.914, "count": 38, "max_ms": 45.18, "p95_ms": 43.334}, "gstreamer_push": {"avg_ms": 1.032, "count": 38, "max_ms": 1.215, "p95_ms": 1.172}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.653, "count": 38, "max_ms": 36.773, "p95_ms": 36.207}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.168, "count": 38, "max_ms": 92.003, "p95_ms": 91.962}, "yolo11_640_fire_11class": {"avg_ms": 48.113, "count": 38, "max_ms": 63.659, "p95_ms": 62.405}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.404, "count": 38, "max_ms": 81.001, "p95_ms": 79.466}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.322, "count": 38, "max_ms": 184.383, "p95_ms": 177.417}, "read_interval": {"avg_ms": 696.384, "count": 37, "max_ms": 10162.963, "p95_ms": 10024.765}, "triton_wait": {"avg_ms": 68.18, "count": 38, "max_ms": 92.014, "p95_ms": 91.974}, "visualize": {"avg_ms": 4.563, "count": 38, "max_ms": 6.287, "p95_ms": 5.483}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11325}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10679.922, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:42:04+09:00", "last_error_summary": "[h264 @ 0x5575bba34540] cabac decode of qscale diff failed at 71 28 [h264 @ 0x5575bba34540] error while decoding MB 71 28, bytestream 138797 [h264 @ 0x5575bba50480] error while decoding MB 68 41, bytestream -8 [in#0/rtsp @ 0x5575bb90ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5575bb90ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 412, "h264_decode_error_count": 1299, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:42:04+09:00", "last_error_summary": "[h264 @ 0x5575bba34540] cabac decode of qscale diff failed at 71 28 [h264 @ 0x5575bba34540] error while decoding MB 71 28, bytestream 138797 [h264 @ 0x5575bba50480] error while decoding MB 68 41, bytestream -8 [in#0/rtsp @ 0x5575bb90ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5575bb90ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:44:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.694, "last_started_at": "2026-07-01T06:42:07+09:00", "pid": 84770, "process": {"exists": false, "pid": 84770}, "restart_count": 451, "restart_count_per_hour": 25, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 452}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.865, "sample_count": 1608, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1359112, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:44:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T06:44:19+09:00", "last_frame_read_age_sec": 0.168, "last_frame_read_at": "2026-07-01T06:44:19+09:00", "mjpeg": {"age_sec": 0.158, "exists": true, "mtime": "2026-07-01T06:44:19.735162+09:00", "mtime_epoch": 1782855859.7351615, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186961}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:44:19+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5013.469, "name": "celery", "pid": 267, "rss_mb": 5013.469, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 161, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.412, "pushed": 5.412, "read": 5.378}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.236, "count": 161, "max_ms": 41.602, "p95_ms": 36.756}, "gstreamer_push": {"avg_ms": 0.937, "count": 162, "max_ms": 1.187, "p95_ms": 1.114}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.453, "count": 161, "max_ms": 38.857, "p95_ms": 35.469}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.254, "count": 161, "max_ms": 93.113, "p95_ms": 88.59}, "yolo11_640_fire_11class": {"avg_ms": 47.771, "count": 161, "max_ms": 65.395, "p95_ms": 61.287}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.137, "count": 161, "max_ms": 85.478, "p95_ms": 80.335}}, "truncated_count": 0}, "processing_total": {"avg_ms": 175.495, "count": 162, "max_ms": 227.655, "p95_ms": 211.453}, "read_interval": {"avg_ms": 186.199, "count": 161, "max_ms": 238.523, "p95_ms": 221.39}, "triton_wait": {"avg_ms": 68.361, "count": 161, "max_ms": 93.125, "p95_ms": 88.602}, "visualize": {"avg_ms": 5.608, "count": 161, "max_ms": 7.677, "p95_ms": 6.998}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11325}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10679.914, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 412, "h264_decode_error_count": 1299, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:42:04+09:00", "last_error_summary": "[h264 @ 0x5575bba34540] cabac decode of qscale diff failed at 71 28 [h264 @ 0x5575bba34540] error while decoding MB 71 28, bytestream 138797 [h264 @ 0x5575bba50480] error while decoding MB 68 41, bytestream -8 [in#0/rtsp @ 0x5575bb90ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5575bb90ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:44:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.698, "last_started_at": "2026-07-01T06:42:07+09:00", "pid": 84770, "process": {"exists": false, "pid": 84770}, "restart_count": 451, "restart_count_per_hour": 25, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 452}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.865, "sample_count": 1608, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1359112, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:44:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T06:44:19+09:00", "last_frame_read_age_sec": 0.171, "last_frame_read_at": "2026-07-01T06:44:19+09:00", "mjpeg": {"age_sec": 0.162, "exists": true, "mtime": "2026-07-01T06:44:19.735162+09:00", "mtime_epoch": 1782855859.7351615, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186961}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:44:19+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5013.469, "name": "celery", "pid": 267, "rss_mb": 5013.469, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 161, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.411, "pushed": 5.411, "read": 5.378}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.236, "count": 161, "max_ms": 41.602, "p95_ms": 36.756}, "gstreamer_push": {"avg_ms": 0.937, "count": 162, "max_ms": 1.187, "p95_ms": 1.114}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.453, "count": 161, "max_ms": 38.857, "p95_ms": 35.469}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.254, "count": 161, "max_ms": 93.113, "p95_ms": 88.59}, "yolo11_640_fire_11class": {"avg_ms": 47.771, "count": 161, "max_ms": 65.395, "p95_ms": 61.287}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.137, "count": 161, "max_ms": 85.478, "p95_ms": 80.335}}, "truncated_count": 0}, "processing_total": {"avg_ms": 175.495, "count": 162, "max_ms": 227.655, "p95_ms": 211.453}, "read_interval": {"avg_ms": 186.199, "count": 161, "max_ms": 238.523, "p95_ms": 221.39}, "triton_wait": {"avg_ms": 68.361, "count": 161, "max_ms": 93.125, "p95_ms": 88.602}, "visualize": {"avg_ms": 5.608, "count": 161, "max_ms": 7.677, "p95_ms": 6.998}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11325}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10788.156, "limit_mb": 8796093022207.996}, "detail": {"pid": 84877, "restart_count": 452, "start_count": 453}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 412, "h264_decode_error_count": 1299, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:42:04+09:00", "last_error_summary": "[h264 @ 0x5575bba34540] cabac decode of qscale diff failed at 71 28 [h264 @ 0x5575bba34540] error while decoding MB 71 28, bytestream 138797 [h264 @ 0x5575bba50480] error while decoding MB 68 41, bytestream -8 [in#0/rtsp @ 0x5575bb90ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5575bb90ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:44:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:44:22+09:00", "pid": 84877, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.992, "name": "ffmpeg", "pid": 84877, "rss_mb": 138.992, "state": "S (sleeping)", "threads": 106}, "restart_count": 452, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 453}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.764, "sample_count": 1599, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1359112, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:44:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.765, "last_frame_pushed_at": "2026-07-01T06:44:19+09:00", "last_frame_read_age_sec": 4.929, "last_frame_read_at": "2026-07-01T06:44:19+09:00", "mjpeg": {"age_sec": 4.748, "exists": true, "mtime": "2026-07-01T06:44:19.907163+09:00", "mtime_epoch": 1782855859.9071631, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187465}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:44:24+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5013.469, "name": "celery", "pid": 267, "rss_mb": 5013.469, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.61, "pushed": 4.61, "read": 4.576}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.222, "count": 137, "max_ms": 41.602, "p95_ms": 36.881}, "gstreamer_push": {"avg_ms": 0.945, "count": 138, "max_ms": 1.187, "p95_ms": 1.124}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.514, "count": 137, "max_ms": 38.857, "p95_ms": 35.469}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.119, "count": 137, "max_ms": 92.007, "p95_ms": 88.59}, "yolo11_640_fire_11class": {"avg_ms": 47.587, "count": 137, "max_ms": 65.395, "p95_ms": 61.587}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.53, "count": 137, "max_ms": 85.478, "p95_ms": 79.81}}, "truncated_count": 0}, "processing_total": {"avg_ms": 173.31, "count": 138, "max_ms": 227.655, "p95_ms": 209.64}, "read_interval": {"avg_ms": 184.055, "count": 137, "max_ms": 238.523, "p95_ms": 219.282}, "triton_wait": {"avg_ms": 68.215, "count": 137, "max_ms": 92.02, "p95_ms": 88.602}, "visualize": {"avg_ms": 5.517, "count": 137, "max_ms": 7.677, "p95_ms": 7.01}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12657}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10606.816, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55813968f600] error while decoding MB 118 39, bytestream -9 [in#0/rtsp @ 0x558139575e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558139575e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 198, "h264_decode_error_count": 203, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:45:06+09:00", "last_error_summary": "[h264 @ 0x55813968f600] error while decoding MB 118 39, bytestream -9 [in#0/rtsp @ 0x558139575e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558139575e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T05:49:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 3341.961, "last_started_at": "2026-07-01T05:49:24+09:00", "pid": 81737, "process": {"exists": false, "pid": 81737}, "restart_count": 197, "restart_count_per_hour": 1, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 198}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.729, "sample_count": 2582, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1518883, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:45:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.082, "last_frame_pushed_at": "2026-07-01T06:45:06+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T06:45:06+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T06:45:06.411635+09:00", "mtime_epoch": 1782855906.4116347, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217506}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T06:45:06+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1778.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.704, "pushed": 0.704, "read": 0.758}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.458, "count": 13, "max_ms": 19.66, "p95_ms": 19.66}, "gstreamer_push": {"avg_ms": 1.188, "count": 13, "max_ms": 1.238, "p95_ms": 1.238}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.907, "count": 13, "max_ms": 50.453, "p95_ms": 50.453}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.634, "count": 13, "max_ms": 74.074, "p95_ms": 74.074}, "yolo11_640_fire_11class": {"avg_ms": 46.896, "count": 13, "max_ms": 51.262, "p95_ms": 51.262}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.365, "count": 13, "max_ms": 64.274, "p95_ms": 64.274}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.012, "count": 13, "max_ms": 101.149, "p95_ms": 101.149}, "read_interval": {"avg_ms": 2660.155, "count": 14, "max_ms": 18834.634, "p95_ms": 18834.634}, "triton_wait": {"avg_ms": 55.646, "count": 13, "max_ms": 74.09, "p95_ms": 74.09}, "visualize": {"avg_ms": 3.392, "count": 13, "max_ms": 3.548, "p95_ms": 3.548}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.476, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12657}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10606.555, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:45:06+09:00", "last_error_summary": "[h264 @ 0x55813968f600] error while decoding MB 118 39, bytestream -9 [in#0/rtsp @ 0x558139575e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558139575e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 198, "h264_decode_error_count": 203, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:45:06+09:00", "last_error_summary": "[h264 @ 0x55813968f600] error while decoding MB 118 39, bytestream -9 [in#0/rtsp @ 0x558139575e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558139575e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:45:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 3341.983, "last_started_at": "2026-07-01T05:49:24+09:00", "pid": 81737, "process": {"exists": false, "pid": 81737}, "restart_count": 197, "restart_count_per_hour": 1, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 198}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.729, "sample_count": 2583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1518884, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:45:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T06:45:06+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-01T06:45:06+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-01T06:45:06.411635+09:00", "mtime_epoch": 1782855906.4116347, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217506}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T06:45:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1778.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.757, "pushed": 0.757, "read": 0.757}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.605, "count": 14, "max_ms": 20.521, "p95_ms": 20.521}, "gstreamer_push": {"avg_ms": 1.198, "count": 14, "max_ms": 1.326, "p95_ms": 1.326}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.576, "count": 14, "max_ms": 50.453, "p95_ms": 50.453}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.359, "count": 14, "max_ms": 74.074, "p95_ms": 74.074}, "yolo11_640_fire_11class": {"avg_ms": 46.788, "count": 14, "max_ms": 51.262, "p95_ms": 51.262}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.113, "count": 14, "max_ms": 64.274, "p95_ms": 64.274}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.063, "count": 14, "max_ms": 101.149, "p95_ms": 101.149}, "read_interval": {"avg_ms": 2660.155, "count": 14, "max_ms": 18834.634, "p95_ms": 18834.634}, "triton_wait": {"avg_ms": 55.372, "count": 14, "max_ms": 74.09, "p95_ms": 74.09}, "visualize": {"avg_ms": 3.48, "count": 14, "max_ms": 4.625, "p95_ms": 4.625}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.498, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12657}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10606.379, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 198, "h264_decode_error_count": 203, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:45:06+09:00", "last_error_summary": "[h264 @ 0x55813968f600] error while decoding MB 118 39, bytestream -9 [in#0/rtsp @ 0x558139575e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558139575e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:45:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 3341.991, "last_started_at": "2026-07-01T05:49:24+09:00", "pid": 81737, "process": {"exists": false, "pid": 81737}, "restart_count": 197, "restart_count_per_hour": 1, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 198}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.729, "sample_count": 2583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1518884, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:45:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T06:45:06+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-01T06:45:06+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T06:45:06.411635+09:00", "mtime_epoch": 1782855906.4116347, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217506}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T06:45:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1778.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.757, "pushed": 0.757, "read": 0.757}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.605, "count": 14, "max_ms": 20.521, "p95_ms": 20.521}, "gstreamer_push": {"avg_ms": 1.198, "count": 14, "max_ms": 1.326, "p95_ms": 1.326}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.576, "count": 14, "max_ms": 50.453, "p95_ms": 50.453}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.359, "count": 14, "max_ms": 74.074, "p95_ms": 74.074}, "yolo11_640_fire_11class": {"avg_ms": 46.788, "count": 14, "max_ms": 51.262, "p95_ms": 51.262}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.113, "count": 14, "max_ms": 64.274, "p95_ms": 64.274}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.063, "count": 14, "max_ms": 101.149, "p95_ms": 101.149}, "read_interval": {"avg_ms": 2660.155, "count": 14, "max_ms": 18834.634, "p95_ms": 18834.634}, "triton_wait": {"avg_ms": 55.372, "count": 14, "max_ms": 74.09, "p95_ms": 74.09}, "visualize": {"avg_ms": 3.48, "count": 14, "max_ms": 4.625, "p95_ms": 4.625}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.506, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 49, "tracking_summary": 12657}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10628.371, "limit_mb": 8796093022207.996}, "detail": {"pid": 84984, "restart_count": 198, "start_count": 199}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 198, "h264_decode_error_count": 203, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:45:06+09:00", "last_error_summary": "[h264 @ 0x55813968f600] error while decoding MB 118 39, bytestream -9 [in#0/rtsp @ 0x558139575e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558139575e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T06:45:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:45:08+09:00", "pid": 84984, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.5, "name": "ffmpeg", "pid": 84984, "rss_mb": 58.5, "state": "S (sleeping)", "threads": 1}, "restart_count": 198, "restart_count_per_hour": 2, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 199}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.676, "sample_count": 2539, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1518884, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:45:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.366, "last_frame_pushed_at": "2026-07-01T06:45:06+09:00", "last_frame_read_age_sec": 4.451, "last_frame_read_at": "2026-07-01T06:45:06+09:00", "mjpeg": {"age_sec": 4.348, "exists": true, "mtime": "2026-07-01T06:45:06.507636+09:00", "mtime_epoch": 1782855906.5076358, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 214535}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T06:45:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1859.934, "name": "celery", "pid": 237, "rss_mb": 1778.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.612, "pushed": 0.612, "read": 0.612}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 18.605, "count": 14, "max_ms": 20.521, "p95_ms": 20.521}, "gstreamer_push": {"avg_ms": 1.198, "count": 14, "max_ms": 1.326, "p95_ms": 1.326}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.576, "count": 14, "max_ms": 50.453, "p95_ms": 50.453}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.359, "count": 14, "max_ms": 74.074, "p95_ms": 74.074}, "yolo11_640_fire_11class": {"avg_ms": 46.788, "count": 14, "max_ms": 51.262, "p95_ms": 51.262}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.113, "count": 14, "max_ms": 64.274, "p95_ms": 64.274}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.063, "count": 14, "max_ms": 101.149, "p95_ms": 101.149}, "read_interval": {"avg_ms": 2660.155, "count": 14, "max_ms": 18834.634, "p95_ms": 18834.634}, "triton_wait": {"avg_ms": 55.372, "count": 14, "max_ms": 74.09, "p95_ms": 74.09}, "visualize": {"avg_ms": 3.48, "count": 14, "max_ms": 4.625, "p95_ms": 4.625}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.859, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11329}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10675.152, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561b552f3080] error while decoding MB 37 12, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 413, "h264_decode_error_count": 1300, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:46:05+09:00", "last_error_summary": "[h264 @ 0x561b552f3080] error while decoding MB 37 12, bytestream -9", "last_exit_at": "2026-07-01T06:44:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 102.602, "last_started_at": "2026-07-01T06:44:22+09:00", "pid": 84877, "process": {"exists": false, "pid": 84877}, "restart_count": 452, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 453}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 230.606, "sample_count": 1485, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1359585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:46:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T06:46:05+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T06:46:05+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T06:46:05.212252+09:00", "mtime_epoch": 1782855965.2122517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184839}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:46:05+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5013.727, "name": "celery", "pid": 267, "rss_mb": 5013.727, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 162, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.41, "pushed": 5.41, "read": 5.41}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.742, "count": 161, "max_ms": 40.364, "p95_ms": 37.844}, "gstreamer_push": {"avg_ms": 0.954, "count": 162, "max_ms": 1.62, "p95_ms": 1.105}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.288, "count": 161, "max_ms": 35.847, "p95_ms": 33.456}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.274, "count": 161, "max_ms": 94.039, "p95_ms": 87.265}, "yolo11_640_fire_11class": {"avg_ms": 46.687, "count": 161, "max_ms": 64.771, "p95_ms": 61.495}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.614, "count": 161, "max_ms": 86.589, "p95_ms": 75.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 176.354, "count": 162, "max_ms": 254.622, "p95_ms": 214.324}, "read_interval": {"avg_ms": 185.82, "count": 162, "max_ms": 264.439, "p95_ms": 224.249}, "triton_wait": {"avg_ms": 65.316, "count": 161, "max_ms": 94.052, "p95_ms": 87.277}, "visualize": {"avg_ms": 5.868, "count": 161, "max_ms": 8.878, "p95_ms": 7.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11329}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10674.512, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:46:05+09:00", "last_error_summary": "[h264 @ 0x561b552f3080] error while decoding MB 37 12, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 413, "h264_decode_error_count": 1300, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:46:05+09:00", "last_error_summary": "[h264 @ 0x561b552f3080] error while decoding MB 37 12, bytestream -9", "last_exit_at": "2026-07-01T06:46:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 102.712, "last_started_at": "2026-07-01T06:44:22+09:00", "pid": 84877, "process": {"exists": false, "pid": 84877}, "restart_count": 452, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 453}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 230.606, "sample_count": 1485, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1359586, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:46:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T06:46:05+09:00", "last_frame_read_age_sec": 0.16, "last_frame_read_at": "2026-07-01T06:46:05+09:00", "mjpeg": {"age_sec": 0.153, "exists": true, "mtime": "2026-07-01T06:46:05.212252+09:00", "mtime_epoch": 1782855965.2122517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184839}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:46:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5013.727, "name": "celery", "pid": 267, "rss_mb": 5013.727, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 161, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.426, "pushed": 5.426, "read": 5.392}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.746, "count": 162, "max_ms": 40.364, "p95_ms": 37.844}, "gstreamer_push": {"avg_ms": 0.955, "count": 162, "max_ms": 1.62, "p95_ms": 1.105}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.297, "count": 162, "max_ms": 35.847, "p95_ms": 33.456}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.234, "count": 162, "max_ms": 94.039, "p95_ms": 87.265}, "yolo11_640_fire_11class": {"avg_ms": 46.677, "count": 162, "max_ms": 64.771, "p95_ms": 61.495}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.611, "count": 162, "max_ms": 86.589, "p95_ms": 75.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 176.025, "count": 162, "max_ms": 254.622, "p95_ms": 214.324}, "read_interval": {"avg_ms": 185.618, "count": 161, "max_ms": 264.439, "p95_ms": 224.249}, "triton_wait": {"avg_ms": 65.275, "count": 162, "max_ms": 94.052, "p95_ms": 87.277}, "visualize": {"avg_ms": 5.864, "count": 162, "max_ms": 8.878, "p95_ms": 7.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.857, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11329}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10674.625, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 413, "h264_decode_error_count": 1300, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:46:05+09:00", "last_error_summary": "[h264 @ 0x561b552f3080] error while decoding MB 37 12, bytestream -9", "last_exit_at": "2026-07-01T06:46:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 102.716, "last_started_at": "2026-07-01T06:44:22+09:00", "pid": 84877, "process": {"exists": false, "pid": 84877}, "restart_count": 452, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 453}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 230.606, "sample_count": 1485, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1359586, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:46:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T06:46:05+09:00", "last_frame_read_age_sec": 0.164, "last_frame_read_at": "2026-07-01T06:46:05+09:00", "mjpeg": {"age_sec": 0.157, "exists": true, "mtime": "2026-07-01T06:46:05.212252+09:00", "mtime_epoch": 1782855965.2122517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184839}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:46:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5013.727, "name": "celery", "pid": 267, "rss_mb": 5013.727, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 161, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.425, "pushed": 5.425, "read": 5.392}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.746, "count": 162, "max_ms": 40.364, "p95_ms": 37.844}, "gstreamer_push": {"avg_ms": 0.955, "count": 162, "max_ms": 1.62, "p95_ms": 1.105}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.297, "count": 162, "max_ms": 35.847, "p95_ms": 33.456}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.234, "count": 162, "max_ms": 94.039, "p95_ms": 87.265}, "yolo11_640_fire_11class": {"avg_ms": 46.677, "count": 162, "max_ms": 64.771, "p95_ms": 61.495}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.611, "count": 162, "max_ms": 86.589, "p95_ms": 75.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 176.025, "count": 162, "max_ms": 254.622, "p95_ms": 214.324}, "read_interval": {"avg_ms": 185.618, "count": 161, "max_ms": 264.439, "p95_ms": 224.249}, "triton_wait": {"avg_ms": 65.275, "count": 162, "max_ms": 94.052, "p95_ms": 87.277}, "visualize": {"avg_ms": 5.864, "count": 162, "max_ms": 8.878, "p95_ms": 7.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.86, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11329}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10782.25, "limit_mb": 8796093022207.996}, "detail": {"pid": 85136, "restart_count": 453, "start_count": 454}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 413, "h264_decode_error_count": 1300, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:46:05+09:00", "last_error_summary": "[h264 @ 0x561b552f3080] error while decoding MB 37 12, bytestream -9", "last_exit_at": "2026-07-01T06:46:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:46:07+09:00", "pid": 85136, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.637, "name": "ffmpeg", "pid": 85136, "rss_mb": 138.637, "state": "S (sleeping)", "threads": 106}, "restart_count": 453, "restart_count_per_hour": 27, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 454}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 230.621, "sample_count": 1454, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1359586, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:46:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.448, "last_frame_pushed_at": "2026-07-01T06:46:05+09:00", "last_frame_read_age_sec": 4.605, "last_frame_read_at": "2026-07-01T06:46:05+09:00", "mjpeg": {"age_sec": 4.434, "exists": true, "mtime": "2026-07-01T06:46:05.376253+09:00", "mtime_epoch": 1782855965.3762534, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173743}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:46:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5013.727, "name": "celery", "pid": 267, "rss_mb": 5013.727, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.703, "pushed": 4.703, "read": 4.67}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.591, "count": 140, "max_ms": 39.887, "p95_ms": 37.844}, "gstreamer_push": {"avg_ms": 0.962, "count": 141, "max_ms": 1.62, "p95_ms": 1.107}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.004, "count": 140, "max_ms": 35.847, "p95_ms": 33.259}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.139, "count": 140, "max_ms": 94.039, "p95_ms": 87.688}, "yolo11_640_fire_11class": {"avg_ms": 46.713, "count": 140, "max_ms": 64.771, "p95_ms": 61.754}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.202, "count": 140, "max_ms": 86.589, "p95_ms": 74.58}}, "truncated_count": 0}, "processing_total": {"avg_ms": 173.382, "count": 141, "max_ms": 254.622, "p95_ms": 202.961}, "read_interval": {"avg_ms": 182.893, "count": 140, "max_ms": 264.439, "p95_ms": 212.759}, "triton_wait": {"avg_ms": 65.185, "count": 140, "max_ms": 94.052, "p95_ms": 87.7}, "visualize": {"avg_ms": 5.744, "count": 140, "max_ms": 8.878, "p95_ms": 7.139}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11338}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10676.066, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c38f99c1c0] error while decoding MB 8 7, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 414, "h264_decode_error_count": 1301, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:48:47+09:00", "last_error_summary": "[h264 @ 0x55c38f99c1c0] error while decoding MB 8 7, bytestream -7", "last_exit_at": "2026-07-01T06:46:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.585, "last_started_at": "2026-07-01T06:46:07+09:00", "pid": 85136, "process": {"exists": false, "pid": 85136}, "restart_count": 453, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 454}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.09, "sample_count": 1684, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1360620, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:48:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T06:48:47+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T06:48:47+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T06:48:47.350053+09:00", "mtime_epoch": 1782856127.350053, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184088}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:48:47+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5014.242, "name": "celery", "pid": 267, "rss_mb": 5014.242, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 198, "pushed_frames": 198, "read_frames": 198, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.616, "pushed": 6.616, "read": 6.616}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.449, "count": 198, "max_ms": 37.13, "p95_ms": 34.135}, "gstreamer_push": {"avg_ms": 1.087, "count": 198, "max_ms": 1.196, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.724, "count": 198, "max_ms": 39.349, "p95_ms": 34.426}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.004, "count": 198, "max_ms": 90.535, "p95_ms": 88.6}, "yolo11_640_fire_11class": {"avg_ms": 51.246, "count": 198, "max_ms": 64.279, "p95_ms": 63.111}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.685, "count": 198, "max_ms": 80.291, "p95_ms": 78.434}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.418, "count": 198, "max_ms": 162.016, "p95_ms": 159.323}, "read_interval": {"avg_ms": 151.633, "count": 198, "max_ms": 174.143, "p95_ms": 170.779}, "triton_wait": {"avg_ms": 71.137, "count": 198, "max_ms": 90.547, "p95_ms": 88.612}, "visualize": {"avg_ms": 4.254, "count": 198, "max_ms": 5.08, "p95_ms": 4.572}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11338}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10675.598, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:48:47+09:00", "last_error_summary": "[h264 @ 0x55c38f99c1c0] error while decoding MB 8 7, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 414, "h264_decode_error_count": 1301, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:48:47+09:00", "last_error_summary": "[h264 @ 0x55c38f99c1c0] error while decoding MB 8 7, bytestream -7", "last_exit_at": "2026-07-01T06:48:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.672, "last_started_at": "2026-07-01T06:46:07+09:00", "pid": 85136, "process": {"exists": false, "pid": 85136}, "restart_count": 453, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 454}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.09, "sample_count": 1684, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1360621, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:48:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T06:48:47+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-01T06:48:47+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-01T06:48:47.350053+09:00", "mtime_epoch": 1782856127.350053, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184088}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:48:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5014.242, "name": "celery", "pid": 267, "rss_mb": 5014.242, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 198, "pushed_frames": 198, "read_frames": 198, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.6, "pushed": 6.6, "read": 6.6}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.469, "count": 198, "max_ms": 37.13, "p95_ms": 34.135}, "gstreamer_push": {"avg_ms": 1.088, "count": 198, "max_ms": 1.222, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.735, "count": 198, "max_ms": 39.349, "p95_ms": 34.426}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.9, "count": 198, "max_ms": 90.535, "p95_ms": 88.6}, "yolo11_640_fire_11class": {"avg_ms": 51.222, "count": 198, "max_ms": 64.279, "p95_ms": 63.111}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.628, "count": 198, "max_ms": 80.291, "p95_ms": 78.434}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.334, "count": 198, "max_ms": 162.016, "p95_ms": 159.323}, "read_interval": {"avg_ms": 151.633, "count": 198, "max_ms": 174.143, "p95_ms": 170.779}, "triton_wait": {"avg_ms": 71.034, "count": 198, "max_ms": 90.547, "p95_ms": 88.612}, "visualize": {"avg_ms": 4.253, "count": 198, "max_ms": 5.08, "p95_ms": 4.572}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11338}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10675.586, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 414, "h264_decode_error_count": 1301, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:48:47+09:00", "last_error_summary": "[h264 @ 0x55c38f99c1c0] error while decoding MB 8 7, bytestream -7", "last_exit_at": "2026-07-01T06:48:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.677, "last_started_at": "2026-07-01T06:46:07+09:00", "pid": 85136, "process": {"exists": false, "pid": 85136}, "restart_count": 453, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 454}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.09, "sample_count": 1684, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1360621, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:48:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T06:48:47+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-01T06:48:47+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-01T06:48:47.350053+09:00", "mtime_epoch": 1782856127.350053, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184088}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:48:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5014.242, "name": "celery", "pid": 267, "rss_mb": 5014.242, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 198, "pushed_frames": 198, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.635, "pushed": 6.635, "read": 6.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.469, "count": 198, "max_ms": 37.13, "p95_ms": 34.135}, "gstreamer_push": {"avg_ms": 1.088, "count": 198, "max_ms": 1.222, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.735, "count": 198, "max_ms": 39.349, "p95_ms": 34.426}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.9, "count": 198, "max_ms": 90.535, "p95_ms": 88.6}, "yolo11_640_fire_11class": {"avg_ms": 51.222, "count": 198, "max_ms": 64.279, "p95_ms": 63.111}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.628, "count": 198, "max_ms": 80.291, "p95_ms": 78.434}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.334, "count": 198, "max_ms": 162.016, "p95_ms": 159.323}, "read_interval": {"avg_ms": 151.581, "count": 197, "max_ms": 174.143, "p95_ms": 170.779}, "triton_wait": {"avg_ms": 71.034, "count": 198, "max_ms": 90.547, "p95_ms": 88.612}, "visualize": {"avg_ms": 4.253, "count": 198, "max_ms": 5.08, "p95_ms": 4.572}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.843, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 438, "tracking_summary": 11338}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10795.055, "limit_mb": 8796093022207.996}, "detail": {"pid": 85243, "restart_count": 454, "start_count": 455}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 414, "h264_decode_error_count": 1301, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:48:47+09:00", "last_error_summary": "[h264 @ 0x55c38f99c1c0] error while decoding MB 8 7, bytestream -7", "last_exit_at": "2026-07-01T06:48:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:48:50+09:00", "pid": 85243, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.617, "name": "ffmpeg", "pid": 85243, "rss_mb": 149.617, "state": "S (sleeping)", "threads": 106}, "restart_count": 454, "restart_count_per_hour": 27, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 455}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.141, "sample_count": 1660, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1360621, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:48:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.804, "last_frame_pushed_at": "2026-07-01T06:48:47+09:00", "last_frame_read_age_sec": 4.937, "last_frame_read_at": "2026-07-01T06:48:47+09:00", "mjpeg": {"age_sec": 4.786, "exists": true, "mtime": "2026-07-01T06:48:47.494055+09:00", "mtime_epoch": 1782856127.4940546, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175732}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:48:52+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5014.242, "name": "celery", "pid": 267, "rss_mb": 5014.242, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 167, "pushed_frames": 167, "read_frames": 166, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.579, "pushed": 5.579, "read": 5.545}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.442, "count": 167, "max_ms": 37.13, "p95_ms": 34.129}, "gstreamer_push": {"avg_ms": 1.086, "count": 167, "max_ms": 1.222, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.906, "count": 167, "max_ms": 39.349, "p95_ms": 34.426}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.625, "count": 167, "max_ms": 89.575, "p95_ms": 87.844}, "yolo11_640_fire_11class": {"avg_ms": 51.24, "count": 167, "max_ms": 64.279, "p95_ms": 62.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.647, "count": 167, "max_ms": 80.291, "p95_ms": 78.434}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.942, "count": 167, "max_ms": 162.016, "p95_ms": 157.832}, "read_interval": {"avg_ms": 151.393, "count": 166, "max_ms": 173.846, "p95_ms": 170.111}, "triton_wait": {"avg_ms": 70.728, "count": 167, "max_ms": 89.587, "p95_ms": 87.858}, "visualize": {"avg_ms": 4.252, "count": 167, "max_ms": 5.08, "p95_ms": 4.572}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11346}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10683.375, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b8afe6d1c0] error while decoding MB 67 43, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 415, "h264_decode_error_count": 1302, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:51:22+09:00", "last_error_summary": "[h264 @ 0x55b8afe6d1c0] error while decoding MB 67 43, bytestream -12", "last_exit_at": "2026-07-01T06:48:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.718, "last_started_at": "2026-07-01T06:48:50+09:00", "pid": 85243, "process": {"exists": false, "pid": 85243}, "restart_count": 454, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 455}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.435, "sample_count": 1890, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1361560, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:51:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T06:51:22+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T06:51:22+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T06:51:22.951891+09:00", "mtime_epoch": 1782856282.9518907, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185954}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:51:23+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5014.758, "name": "celery", "pid": 267, "rss_mb": 5014.758, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 157, "pushed_frames": 157, "read_frames": 157, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.238, "pushed": 5.238, "read": 5.238}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.39, "count": 156, "max_ms": 47.258, "p95_ms": 41.116}, "gstreamer_push": {"avg_ms": 0.959, "count": 157, "max_ms": 1.243, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.083, "count": 156, "max_ms": 38.415, "p95_ms": 36.159}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.027, "count": 156, "max_ms": 92.882, "p95_ms": 85.549}, "yolo11_640_fire_11class": {"avg_ms": 46.266, "count": 156, "max_ms": 69.989, "p95_ms": 59.329}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.587, "count": 156, "max_ms": 86.731, "p95_ms": 77.287}}, "truncated_count": 0}, "processing_total": {"avg_ms": 181.236, "count": 157, "max_ms": 871.611, "p95_ms": 214.142}, "read_interval": {"avg_ms": 191.592, "count": 157, "max_ms": 881.817, "p95_ms": 222.907}, "triton_wait": {"avg_ms": 65.045, "count": 156, "max_ms": 92.894, "p95_ms": 85.56}, "visualize": {"avg_ms": 5.466, "count": 156, "max_ms": 10.337, "p95_ms": 9.613}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11346}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10682.773, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:51:22+09:00", "last_error_summary": "[h264 @ 0x55b8afe6d1c0] error while decoding MB 67 43, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 415, "h264_decode_error_count": 1302, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:51:22+09:00", "last_error_summary": "[h264 @ 0x55b8afe6d1c0] error while decoding MB 67 43, bytestream -12", "last_exit_at": "2026-07-01T06:51:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.862, "last_started_at": "2026-07-01T06:48:50+09:00", "pid": 85243, "process": {"exists": false, "pid": 85243}, "restart_count": 454, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 455}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.73, "sample_count": 1890, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1361561, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:51:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T06:51:23+09:00", "last_frame_read_age_sec": 0.197, "last_frame_read_at": "2026-07-01T06:51:22+09:00", "mjpeg": {"age_sec": 0.189, "exists": true, "mtime": "2026-07-01T06:51:22.951891+09:00", "mtime_epoch": 1782856282.9518907, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185954}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:51:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5014.758, "name": "celery", "pid": 267, "rss_mb": 5014.758, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 157, "pushed_frames": 157, "read_frames": 156, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.238, "pushed": 5.238, "read": 5.205}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.399, "count": 156, "max_ms": 47.258, "p95_ms": 41.116}, "gstreamer_push": {"avg_ms": 0.957, "count": 157, "max_ms": 1.243, "p95_ms": 1.106}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.076, "count": 156, "max_ms": 38.415, "p95_ms": 36.159}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.083, "count": 156, "max_ms": 92.882, "p95_ms": 85.549}, "yolo11_640_fire_11class": {"avg_ms": 46.302, "count": 156, "max_ms": 69.989, "p95_ms": 59.329}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.646, "count": 156, "max_ms": 86.731, "p95_ms": 77.287}}, "truncated_count": 0}, "processing_total": {"avg_ms": 181.454, "count": 157, "max_ms": 871.611, "p95_ms": 214.142}, "read_interval": {"avg_ms": 191.736, "count": 156, "max_ms": 881.817, "p95_ms": 222.907}, "triton_wait": {"avg_ms": 65.1, "count": 156, "max_ms": 92.894, "p95_ms": 85.56}, "visualize": {"avg_ms": 5.471, "count": 156, "max_ms": 10.337, "p95_ms": 9.613}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11346}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10682.891, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 415, "h264_decode_error_count": 1302, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:51:22+09:00", "last_error_summary": "[h264 @ 0x55b8afe6d1c0] error while decoding MB 67 43, bytestream -12", "last_exit_at": "2026-07-01T06:51:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.866, "last_started_at": "2026-07-01T06:48:50+09:00", "pid": 85243, "process": {"exists": false, "pid": 85243}, "restart_count": 454, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 455}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.73, "sample_count": 1890, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1361561, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:51:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T06:51:23+09:00", "last_frame_read_age_sec": 0.201, "last_frame_read_at": "2026-07-01T06:51:22+09:00", "mjpeg": {"age_sec": 0.193, "exists": true, "mtime": "2026-07-01T06:51:22.951891+09:00", "mtime_epoch": 1782856282.9518907, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185954}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:51:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5014.758, "name": "celery", "pid": 267, "rss_mb": 5014.758, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 157, "pushed_frames": 157, "read_frames": 156, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.237, "pushed": 5.237, "read": 5.204}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.399, "count": 156, "max_ms": 47.258, "p95_ms": 41.116}, "gstreamer_push": {"avg_ms": 0.957, "count": 157, "max_ms": 1.243, "p95_ms": 1.106}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.076, "count": 156, "max_ms": 38.415, "p95_ms": 36.159}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.083, "count": 156, "max_ms": 92.882, "p95_ms": 85.549}, "yolo11_640_fire_11class": {"avg_ms": 46.302, "count": 156, "max_ms": 69.989, "p95_ms": 59.329}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.646, "count": 156, "max_ms": 86.731, "p95_ms": 77.287}}, "truncated_count": 0}, "processing_total": {"avg_ms": 181.454, "count": 157, "max_ms": 871.611, "p95_ms": 214.142}, "read_interval": {"avg_ms": 191.736, "count": 156, "max_ms": 881.817, "p95_ms": 222.907}, "triton_wait": {"avg_ms": 65.1, "count": 156, "max_ms": 92.894, "p95_ms": 85.56}, "visualize": {"avg_ms": 5.471, "count": 156, "max_ms": 10.337, "p95_ms": 9.613}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11346}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10788.48, "limit_mb": 8796093022207.996}, "detail": {"pid": 85410, "restart_count": 455, "start_count": 456}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 415, "h264_decode_error_count": 1302, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:51:22+09:00", "last_error_summary": "[h264 @ 0x55b8afe6d1c0] error while decoding MB 67 43, bytestream -12", "last_exit_at": "2026-07-01T06:51:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:51:25+09:00", "pid": 85410, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.527, "name": "ffmpeg", "pid": 85410, "rss_mb": 136.527, "state": "S (sleeping)", "threads": 106}, "restart_count": 455, "restart_count_per_hour": 27, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 456}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.953, "sample_count": 1860, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1361561, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:51:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.781, "last_frame_pushed_at": "2026-07-01T06:51:23+09:00", "last_frame_read_age_sec": 4.975, "last_frame_read_at": "2026-07-01T06:51:22+09:00", "mjpeg": {"age_sec": 4.763, "exists": true, "mtime": "2026-07-01T06:51:23.155893+09:00", "mtime_epoch": 1782856283.155893, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180800}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:51:27+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5014.758, "name": "celery", "pid": 267, "rss_mb": 5014.758, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.514, "pushed": 4.514, "read": 4.481}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.576, "count": 135, "max_ms": 47.258, "p95_ms": 41.206}, "gstreamer_push": {"avg_ms": 0.96, "count": 135, "max_ms": 1.243, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.872, "count": 135, "max_ms": 38.415, "p95_ms": 35.492}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.959, "count": 135, "max_ms": 92.882, "p95_ms": 83.159}, "yolo11_640_fire_11class": {"avg_ms": 45.993, "count": 135, "max_ms": 69.989, "p95_ms": 58.17}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.529, "count": 135, "max_ms": 86.731, "p95_ms": 74.731}}, "truncated_count": 0}, "processing_total": {"avg_ms": 177.174, "count": 135, "max_ms": 349.997, "p95_ms": 211.985}, "read_interval": {"avg_ms": 187.368, "count": 134, "max_ms": 361.409, "p95_ms": 221.984}, "triton_wait": {"avg_ms": 64.977, "count": 135, "max_ms": 92.894, "p95_ms": 83.17}, "visualize": {"avg_ms": 5.358, "count": 135, "max_ms": 9.682, "p95_ms": 7.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11354}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10684.77, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56443ed41c80] error while decoding MB 28 36, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 416, "h264_decode_error_count": 1303, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:54:05+09:00", "last_error_summary": "[h264 @ 0x56443ed41c80] error while decoding MB 28 36, bytestream -7", "last_exit_at": "2026-07-01T06:51:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.186, "last_started_at": "2026-07-01T06:51:25+09:00", "pid": 85410, "process": {"exists": false, "pid": 85410}, "restart_count": 455, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 456}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.166, "sample_count": 1889, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1362593, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:54:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T06:54:05+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T06:54:05+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T06:54:05.061888+09:00", "mtime_epoch": 1782856445.061888, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187588}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:54:05+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5015.273, "name": "celery", "pid": 267, "rss_mb": 5015.273, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.585, "pushed": 6.585, "read": 6.585}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.772, "count": 197, "max_ms": 33.865, "p95_ms": 32.73}, "gstreamer_push": {"avg_ms": 1.102, "count": 197, "max_ms": 1.506, "p95_ms": 1.167}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.158, "count": 197, "max_ms": 44.878, "p95_ms": 37.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.013, "count": 197, "max_ms": 89.71, "p95_ms": 86.941}, "yolo11_640_fire_11class": {"avg_ms": 52.213, "count": 197, "max_ms": 68.289, "p95_ms": 63.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.086, "count": 197, "max_ms": 85.173, "p95_ms": 78.755}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.849, "count": 197, "max_ms": 162.226, "p95_ms": 156.624}, "read_interval": {"avg_ms": 152.274, "count": 197, "max_ms": 175.812, "p95_ms": 169.019}, "triton_wait": {"avg_ms": 71.182, "count": 197, "max_ms": 89.722, "p95_ms": 86.951}, "visualize": {"avg_ms": 4.337, "count": 197, "max_ms": 5.308, "p95_ms": 4.517}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11354}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10684.402, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:54:05+09:00", "last_error_summary": "[h264 @ 0x56443ed41c80] error while decoding MB 28 36, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 416, "h264_decode_error_count": 1303, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:54:05+09:00", "last_error_summary": "[h264 @ 0x56443ed41c80] error while decoding MB 28 36, bytestream -7", "last_exit_at": "2026-07-01T06:54:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.294, "last_started_at": "2026-07-01T06:51:25+09:00", "pid": 85410, "process": {"exists": false, "pid": 85410}, "restart_count": 455, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 456}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.166, "sample_count": 1889, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1362594, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:54:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T06:54:05+09:00", "last_frame_read_age_sec": 0.158, "last_frame_read_at": "2026-07-01T06:54:05+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-01T06:54:05.061888+09:00", "mtime_epoch": 1782856445.061888, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187588}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:54:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5015.273, "name": "celery", "pid": 267, "rss_mb": 5015.273, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 196, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.594, "pushed": 6.594, "read": 6.561}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.774, "count": 197, "max_ms": 33.865, "p95_ms": 32.73}, "gstreamer_push": {"avg_ms": 1.102, "count": 197, "max_ms": 1.506, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.152, "count": 197, "max_ms": 44.878, "p95_ms": 37.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.107, "count": 197, "max_ms": 89.71, "p95_ms": 86.941}, "yolo11_640_fire_11class": {"avg_ms": 52.3, "count": 197, "max_ms": 68.289, "p95_ms": 63.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.227, "count": 197, "max_ms": 85.173, "p95_ms": 78.937}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.936, "count": 197, "max_ms": 162.226, "p95_ms": 156.624}, "read_interval": {"avg_ms": 152.303, "count": 196, "max_ms": 175.812, "p95_ms": 169.019}, "triton_wait": {"avg_ms": 71.276, "count": 197, "max_ms": 89.722, "p95_ms": 86.951}, "visualize": {"avg_ms": 4.336, "count": 197, "max_ms": 5.308, "p95_ms": 4.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.875, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11354}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10684.395, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 416, "h264_decode_error_count": 1303, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:54:05+09:00", "last_error_summary": "[h264 @ 0x56443ed41c80] error while decoding MB 28 36, bytestream -7", "last_exit_at": "2026-07-01T06:54:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.299, "last_started_at": "2026-07-01T06:51:25+09:00", "pid": 85410, "process": {"exists": false, "pid": 85410}, "restart_count": 455, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 456}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.166, "sample_count": 1889, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1362594, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:54:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T06:54:05+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-01T06:54:05+09:00", "mjpeg": {"age_sec": 0.156, "exists": true, "mtime": "2026-07-01T06:54:05.061888+09:00", "mtime_epoch": 1782856445.061888, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187588}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:54:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5015.273, "name": "celery", "pid": 267, "rss_mb": 5015.273, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 196, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.593, "pushed": 6.593, "read": 6.559}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.774, "count": 197, "max_ms": 33.865, "p95_ms": 32.73}, "gstreamer_push": {"avg_ms": 1.102, "count": 197, "max_ms": 1.506, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.152, "count": 197, "max_ms": 44.878, "p95_ms": 37.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.107, "count": 197, "max_ms": 89.71, "p95_ms": 86.941}, "yolo11_640_fire_11class": {"avg_ms": 52.3, "count": 197, "max_ms": 68.289, "p95_ms": 63.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.227, "count": 197, "max_ms": 85.173, "p95_ms": 78.937}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.936, "count": 197, "max_ms": 162.226, "p95_ms": 156.624}, "read_interval": {"avg_ms": 152.303, "count": 196, "max_ms": 175.812, "p95_ms": 169.019}, "triton_wait": {"avg_ms": 71.276, "count": 197, "max_ms": 89.722, "p95_ms": 86.951}, "visualize": {"avg_ms": 4.336, "count": 197, "max_ms": 5.308, "p95_ms": 4.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11354}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10789.145, "limit_mb": 8796093022207.996}, "detail": {"pid": 85517, "restart_count": 456, "start_count": 457}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 416, "h264_decode_error_count": 1303, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:54:05+09:00", "last_error_summary": "[h264 @ 0x56443ed41c80] error while decoding MB 28 36, bytestream -7", "last_exit_at": "2026-07-01T06:54:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:54:07+09:00", "pid": 85517, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.676, "name": "ffmpeg", "pid": 85517, "rss_mb": 136.676, "state": "S (sleeping)", "threads": 106}, "restart_count": 456, "restart_count_per_hour": 27, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 457}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.266, "sample_count": 1861, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1362594, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:54:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.264, "last_frame_pushed_at": "2026-07-01T06:54:05+09:00", "last_frame_read_age_sec": 4.418, "last_frame_read_at": "2026-07-01T06:54:05+09:00", "mjpeg": {"age_sec": 4.25, "exists": true, "mtime": "2026-07-01T06:54:05.221890+09:00", "mtime_epoch": 1782856445.22189, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187757}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:54:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5015.273, "name": "celery", "pid": 267, "rss_mb": 5015.273, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 170, "pushed_frames": 170, "read_frames": 169, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.677, "pushed": 5.677, "read": 5.644}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.726, "count": 170, "max_ms": 33.865, "p95_ms": 32.73}, "gstreamer_push": {"avg_ms": 1.101, "count": 170, "max_ms": 1.506, "p95_ms": 1.159}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.155, "count": 170, "max_ms": 44.878, "p95_ms": 37.47}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.755, "count": 170, "max_ms": 89.71, "p95_ms": 86.842}, "yolo11_640_fire_11class": {"avg_ms": 52.071, "count": 170, "max_ms": 68.289, "p95_ms": 63.459}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.13, "count": 170, "max_ms": 85.173, "p95_ms": 79.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.591, "count": 170, "max_ms": 162.226, "p95_ms": 156.422}, "read_interval": {"avg_ms": 151.873, "count": 169, "max_ms": 175.812, "p95_ms": 168.591}, "triton_wait": {"avg_ms": 70.949, "count": 170, "max_ms": 89.722, "p95_ms": 86.855}, "visualize": {"avg_ms": 4.332, "count": 170, "max_ms": 5.308, "p95_ms": 4.507}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11360}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10685.75, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55feab49b840] error while decoding MB 20 39, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 417, "h264_decode_error_count": 1304, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:56:11+09:00", "last_error_summary": "[h264 @ 0x55feab49b840] error while decoding MB 20 39, bytestream -11", "last_exit_at": "2026-07-01T06:54:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.25, "last_started_at": "2026-07-01T06:54:07+09:00", "pid": 85517, "process": {"exists": false, "pid": 85517}, "restart_count": 456, "restart_count_per_hour": 27, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 457}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.728, "sample_count": 1777, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1363275, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:56:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T06:56:11+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T06:56:11+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T06:56:11.675492+09:00", "mtime_epoch": 1782856571.675492, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186046}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:56:11+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5015.531, "name": "celery", "pid": 267, "rss_mb": 5015.531, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 196, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.563, "pushed": 6.563, "read": 6.563}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.394, "count": 196, "max_ms": 37.761, "p95_ms": 35.97}, "gstreamer_push": {"avg_ms": 1.096, "count": 196, "max_ms": 1.447, "p95_ms": 1.188}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.98, "count": 196, "max_ms": 42.612, "p95_ms": 36.479}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.406, "count": 196, "max_ms": 91.512, "p95_ms": 87.314}, "yolo11_640_fire_11class": {"avg_ms": 50.587, "count": 196, "max_ms": 67.514, "p95_ms": 65.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.459, "count": 196, "max_ms": 82.463, "p95_ms": 78.676}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.928, "count": 196, "max_ms": 184.592, "p95_ms": 161.126}, "read_interval": {"avg_ms": 152.818, "count": 196, "max_ms": 195.288, "p95_ms": 172.648}, "triton_wait": {"avg_ms": 68.529, "count": 196, "max_ms": 91.525, "p95_ms": 87.325}, "visualize": {"avg_ms": 4.403, "count": 196, "max_ms": 6.136, "p95_ms": 5.143}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.864, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11360}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10685.348, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:56:11+09:00", "last_error_summary": "[h264 @ 0x55feab49b840] error while decoding MB 20 39, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 417, "h264_decode_error_count": 1304, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:56:11+09:00", "last_error_summary": "[h264 @ 0x55feab49b840] error while decoding MB 20 39, bytestream -11", "last_exit_at": "2026-07-01T06:56:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.333, "last_started_at": "2026-07-01T06:54:07+09:00", "pid": 85517, "process": {"exists": false, "pid": 85517}, "restart_count": 456, "restart_count_per_hour": 27, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 457}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.728, "sample_count": 1778, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1363276, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:56:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T06:56:11+09:00", "last_frame_read_age_sec": 0.135, "last_frame_read_at": "2026-07-01T06:56:11+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-01T06:56:11.675492+09:00", "mtime_epoch": 1782856571.675492, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186046}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:56:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5015.531, "name": "celery", "pid": 267, "rss_mb": 5015.531, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 196, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.579, "pushed": 6.579, "read": 6.545}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.395, "count": 197, "max_ms": 37.761, "p95_ms": 35.97}, "gstreamer_push": {"avg_ms": 1.095, "count": 197, "max_ms": 1.447, "p95_ms": 1.188}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.977, "count": 197, "max_ms": 42.612, "p95_ms": 36.479}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.351, "count": 197, "max_ms": 91.512, "p95_ms": 87.314}, "yolo11_640_fire_11class": {"avg_ms": 50.557, "count": 197, "max_ms": 67.514, "p95_ms": 65.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.413, "count": 197, "max_ms": 82.463, "p95_ms": 78.676}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.867, "count": 197, "max_ms": 184.592, "p95_ms": 161.126}, "read_interval": {"avg_ms": 152.818, "count": 196, "max_ms": 195.288, "p95_ms": 172.648}, "triton_wait": {"avg_ms": 68.473, "count": 197, "max_ms": 91.525, "p95_ms": 87.325}, "visualize": {"avg_ms": 4.404, "count": 197, "max_ms": 6.136, "p95_ms": 5.143}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11360}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10685.359, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 417, "h264_decode_error_count": 1304, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:56:11+09:00", "last_error_summary": "[h264 @ 0x55feab49b840] error while decoding MB 20 39, bytestream -11", "last_exit_at": "2026-07-01T06:56:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.338, "last_started_at": "2026-07-01T06:54:07+09:00", "pid": 85517, "process": {"exists": false, "pid": 85517}, "restart_count": 456, "restart_count_per_hour": 27, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 457}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.728, "sample_count": 1778, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1363276, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:56:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T06:56:11+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-01T06:56:11+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-01T06:56:11.675492+09:00", "mtime_epoch": 1782856571.675492, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186046}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:56:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5015.531, "name": "celery", "pid": 267, "rss_mb": 5015.531, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 196, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.578, "pushed": 6.578, "read": 6.544}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.395, "count": 197, "max_ms": 37.761, "p95_ms": 35.97}, "gstreamer_push": {"avg_ms": 1.095, "count": 197, "max_ms": 1.447, "p95_ms": 1.188}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.977, "count": 197, "max_ms": 42.612, "p95_ms": 36.479}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.351, "count": 197, "max_ms": 91.512, "p95_ms": 87.314}, "yolo11_640_fire_11class": {"avg_ms": 50.557, "count": 197, "max_ms": 67.514, "p95_ms": 65.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.413, "count": 197, "max_ms": 82.463, "p95_ms": 78.676}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.867, "count": 197, "max_ms": 184.592, "p95_ms": 161.126}, "read_interval": {"avg_ms": 152.818, "count": 196, "max_ms": 195.288, "p95_ms": 172.648}, "triton_wait": {"avg_ms": 68.473, "count": 197, "max_ms": 91.525, "p95_ms": 87.325}, "visualize": {"avg_ms": 4.404, "count": 197, "max_ms": 6.136, "p95_ms": 5.143}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11360}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10787.973, "limit_mb": 8796093022207.996}, "detail": {"pid": 85624, "restart_count": 457, "start_count": 458}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 417, "h264_decode_error_count": 1304, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:56:11+09:00", "last_error_summary": "[h264 @ 0x55feab49b840] error while decoding MB 20 39, bytestream -11", "last_exit_at": "2026-07-01T06:56:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:56:14+09:00", "pid": 85624, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.918, "name": "ffmpeg", "pid": 85624, "rss_mb": 133.918, "state": "S (sleeping)", "threads": 106}, "restart_count": 457, "restart_count_per_hour": 27, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 458}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.728, "sample_count": 1752, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1363276, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:56:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.64, "last_frame_pushed_at": "2026-07-01T06:56:11+09:00", "last_frame_read_age_sec": 4.771, "last_frame_read_at": "2026-07-01T06:56:11+09:00", "mjpeg": {"age_sec": 4.62, "exists": true, "mtime": "2026-07-01T06:56:11.819494+09:00", "mtime_epoch": 1782856571.819494, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186427}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:56:16+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5015.531, "name": "celery", "pid": 267, "rss_mb": 5015.531, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 166, "pushed_frames": 166, "read_frames": 165, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.541, "pushed": 5.541, "read": 5.508}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.294, "count": 166, "max_ms": 37.31, "p95_ms": 35.334}, "gstreamer_push": {"avg_ms": 1.094, "count": 166, "max_ms": 1.377, "p95_ms": 1.188}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.023, "count": 166, "max_ms": 42.612, "p95_ms": 36.479}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.651, "count": 166, "max_ms": 91.512, "p95_ms": 87.029}, "yolo11_640_fire_11class": {"avg_ms": 50.723, "count": 166, "max_ms": 67.514, "p95_ms": 65.53}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.615, "count": 166, "max_ms": 82.463, "p95_ms": 78.958}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.464, "count": 166, "max_ms": 184.592, "p95_ms": 162.219}, "read_interval": {"avg_ms": 153.435, "count": 165, "max_ms": 195.288, "p95_ms": 172.757}, "triton_wait": {"avg_ms": 68.761, "count": 166, "max_ms": 91.525, "p95_ms": 87.041}, "visualize": {"avg_ms": 4.432, "count": 166, "max_ms": 6.136, "p95_ms": 5.228}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11369}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10686.562, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562bd0c97180] error while decoding MB 39 39, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 418, "h264_decode_error_count": 1305, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:58:57+09:00", "last_error_summary": "[h264 @ 0x562bd0c97180] error while decoding MB 39 39, bytestream -5", "last_exit_at": "2026-07-01T06:56:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.066, "last_started_at": "2026-07-01T06:56:14+09:00", "pid": 85624, "process": {"exists": false, "pid": 85624}, "restart_count": 457, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 458}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.21, "sample_count": 1829, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1364372, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:58:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T06:58:57+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T06:58:57+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T06:58:57.463500+09:00", "mtime_epoch": 1782856737.4635003, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185412}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:58:57+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5016.047, "name": "celery", "pid": 267, "rss_mb": 5016.047, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.854, "pushed": 6.854, "read": 6.854}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.699, "count": 205, "max_ms": 42.63, "p95_ms": 36.762}, "gstreamer_push": {"avg_ms": 1.103, "count": 205, "max_ms": 1.377, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.126, "count": 205, "max_ms": 39.643, "p95_ms": 33.74}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.956, "count": 205, "max_ms": 88.01, "p95_ms": 82.829}, "yolo11_640_fire_11class": {"avg_ms": 48.853, "count": 205, "max_ms": 69.509, "p95_ms": 63.159}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.282, "count": 205, "max_ms": 81.878, "p95_ms": 76.226}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.268, "count": 205, "max_ms": 163.567, "p95_ms": 152.901}, "read_interval": {"avg_ms": 146.28, "count": 205, "max_ms": 177.132, "p95_ms": 165.337}, "triton_wait": {"avg_ms": 64.069, "count": 205, "max_ms": 88.022, "p95_ms": 82.841}, "visualize": {"avg_ms": 4.22, "count": 205, "max_ms": 5.084, "p95_ms": 4.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11369}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10686.625, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:58:57+09:00", "last_error_summary": "[h264 @ 0x562bd0c97180] error while decoding MB 39 39, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 418, "h264_decode_error_count": 1305, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:58:57+09:00", "last_error_summary": "[h264 @ 0x562bd0c97180] error while decoding MB 39 39, bytestream -5", "last_exit_at": "2026-07-01T06:58:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.166, "last_started_at": "2026-07-01T06:56:14+09:00", "pid": 85624, "process": {"exists": false, "pid": 85624}, "restart_count": 457, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 458}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.21, "sample_count": 1829, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1364373, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:58:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T06:58:57+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-01T06:58:57+09:00", "mjpeg": {"age_sec": 0.141, "exists": true, "mtime": "2026-07-01T06:58:57.463500+09:00", "mtime_epoch": 1782856737.4635003, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185412}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:58:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5016.047, "name": "celery", "pid": 267, "rss_mb": 5016.047, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.834, "pushed": 6.834, "read": 6.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.749, "count": 205, "max_ms": 42.63, "p95_ms": 37.291}, "gstreamer_push": {"avg_ms": 1.103, "count": 205, "max_ms": 1.377, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.113, "count": 205, "max_ms": 39.643, "p95_ms": 33.74}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.936, "count": 205, "max_ms": 88.01, "p95_ms": 82.829}, "yolo11_640_fire_11class": {"avg_ms": 48.82, "count": 205, "max_ms": 69.509, "p95_ms": 63.159}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.25, "count": 205, "max_ms": 81.878, "p95_ms": 76.226}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.34, "count": 205, "max_ms": 163.567, "p95_ms": 152.901}, "read_interval": {"avg_ms": 146.28, "count": 205, "max_ms": 177.132, "p95_ms": 165.337}, "triton_wait": {"avg_ms": 64.05, "count": 205, "max_ms": 88.022, "p95_ms": 82.841}, "visualize": {"avg_ms": 4.223, "count": 205, "max_ms": 5.108, "p95_ms": 4.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11369}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10686.594, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 418, "h264_decode_error_count": 1305, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:58:57+09:00", "last_error_summary": "[h264 @ 0x562bd0c97180] error while decoding MB 39 39, bytestream -5", "last_exit_at": "2026-07-01T06:58:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.172, "last_started_at": "2026-07-01T06:56:14+09:00", "pid": 85624, "process": {"exists": false, "pid": 85624}, "restart_count": 457, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 458}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.21, "sample_count": 1829, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1364373, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:58:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T06:58:57+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-01T06:58:57+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-01T06:58:57.463500+09:00", "mtime_epoch": 1782856737.4635003, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185412}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:58:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5016.047, "name": "celery", "pid": 267, "rss_mb": 5016.047, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.863, "pushed": 6.863, "read": 6.83}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.749, "count": 205, "max_ms": 42.63, "p95_ms": 37.291}, "gstreamer_push": {"avg_ms": 1.103, "count": 205, "max_ms": 1.377, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.113, "count": 205, "max_ms": 39.643, "p95_ms": 33.74}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.936, "count": 205, "max_ms": 88.01, "p95_ms": 82.829}, "yolo11_640_fire_11class": {"avg_ms": 48.82, "count": 205, "max_ms": 69.509, "p95_ms": 63.159}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.25, "count": 205, "max_ms": 81.878, "p95_ms": 76.226}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.34, "count": 205, "max_ms": 163.567, "p95_ms": 152.901}, "read_interval": {"avg_ms": 146.316, "count": 204, "max_ms": 177.132, "p95_ms": 165.337}, "triton_wait": {"avg_ms": 64.05, "count": 205, "max_ms": 88.022, "p95_ms": 82.841}, "visualize": {"avg_ms": 4.223, "count": 205, "max_ms": 5.108, "p95_ms": 4.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11369}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10795.098, "limit_mb": 8796093022207.996}, "detail": {"pid": 85731, "restart_count": 458, "start_count": 459}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 418, "h264_decode_error_count": 1305, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:58:57+09:00", "last_error_summary": "[h264 @ 0x562bd0c97180] error while decoding MB 39 39, bytestream -5", "last_exit_at": "2026-07-01T06:59:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T06:59:00+09:00", "pid": 85731, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.23, "name": "ffmpeg", "pid": 85731, "rss_mb": 139.23, "state": "S (sleeping)", "threads": 106}, "restart_count": 458, "restart_count_per_hour": 27, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 459}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.427, "sample_count": 1800, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1364373, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T06:58:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.562, "last_frame_pushed_at": "2026-07-01T06:58:57+09:00", "last_frame_read_age_sec": 4.706, "last_frame_read_at": "2026-07-01T06:58:57+09:00", "mjpeg": {"age_sec": 4.541, "exists": true, "mtime": "2026-07-01T06:58:57.619505+09:00", "mtime_epoch": 1782856737.6195047, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185948}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T06:59:02+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5016.047, "name": "celery", "pid": 267, "rss_mb": 5016.047, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.802, "pushed": 5.802, "read": 5.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.702, "count": 174, "max_ms": 42.63, "p95_ms": 36.569}, "gstreamer_push": {"avg_ms": 1.103, "count": 174, "max_ms": 1.377, "p95_ms": 1.187}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.042, "count": 174, "max_ms": 39.643, "p95_ms": 34.19}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.808, "count": 174, "max_ms": 88.01, "p95_ms": 83.005}, "yolo11_640_fire_11class": {"avg_ms": 48.609, "count": 174, "max_ms": 69.509, "p95_ms": 63.423}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.939, "count": 174, "max_ms": 81.878, "p95_ms": 76.226}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.2, "count": 174, "max_ms": 163.567, "p95_ms": 153.087}, "read_interval": {"avg_ms": 146.156, "count": 174, "max_ms": 177.132, "p95_ms": 165.507}, "triton_wait": {"avg_ms": 63.89, "count": 174, "max_ms": 88.022, "p95_ms": 83.018}, "visualize": {"avg_ms": 4.234, "count": 174, "max_ms": 5.108, "p95_ms": 4.563}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11378}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10689.145, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T06:58:57+09:00", "last_error_summary": "[h264 @ 0x562bd0c97180] error while decoding MB 39 39, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 418, "h264_decode_error_count": 1305, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:58:57+09:00", "last_error_summary": "[h264 @ 0x562bd0c97180] error while decoding MB 39 39, bytestream -5", "last_exit_at": "2026-07-01T07:01:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 154.213, "last_started_at": "2026-07-01T06:59:00+09:00", "pid": 85731, "process": {"exists": false, "pid": 85731}, "restart_count": 458, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 459}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.054, "sample_count": 1978, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1365377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:01:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:01:34+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-01T07:01:34+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-01T07:01:34.247788+09:00", "mtime_epoch": 1782856894.247788, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187478}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:01:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5016.82, "name": "celery", "pid": 267, "rss_mb": 5016.82, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.857, "pushed": 6.857, "read": 6.824}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.213, "count": 205, "max_ms": 49.008, "p95_ms": 35.165}, "gstreamer_push": {"avg_ms": 1.088, "count": 205, "max_ms": 1.7, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.274, "count": 205, "max_ms": 64.646, "p95_ms": 34.93}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.53, "count": 205, "max_ms": 91.338, "p95_ms": 83.331}, "yolo11_640_fire_11class": {"avg_ms": 49.682, "count": 205, "max_ms": 72.396, "p95_ms": 63.59}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.933, "count": 205, "max_ms": 88.616, "p95_ms": 77.509}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.537, "count": 205, "max_ms": 159.835, "p95_ms": 153.347}, "read_interval": {"avg_ms": 146.681, "count": 204, "max_ms": 171.658, "p95_ms": 165.949}, "triton_wait": {"avg_ms": 64.595, "count": 205, "max_ms": 91.352, "p95_ms": 83.344}, "visualize": {"avg_ms": 4.277, "count": 205, "max_ms": 5.37, "p95_ms": 4.676}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11378}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10689.137, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 418, "h264_decode_error_count": 1305, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:58:57+09:00", "last_error_summary": "[h264 @ 0x562bd0c97180] error while decoding MB 39 39, bytestream -5", "last_exit_at": "2026-07-01T07:01:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 154.218, "last_started_at": "2026-07-01T06:59:00+09:00", "pid": 85731, "process": {"exists": false, "pid": 85731}, "restart_count": 458, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 459}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.054, "sample_count": 1978, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1365377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:01:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:01:34+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-01T07:01:34+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T07:01:34.247788+09:00", "mtime_epoch": 1782856894.247788, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187478}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:01:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5016.82, "name": "celery", "pid": 267, "rss_mb": 5016.82, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.856, "pushed": 6.856, "read": 6.823}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.213, "count": 205, "max_ms": 49.008, "p95_ms": 35.165}, "gstreamer_push": {"avg_ms": 1.088, "count": 205, "max_ms": 1.7, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.274, "count": 205, "max_ms": 64.646, "p95_ms": 34.93}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.53, "count": 205, "max_ms": 91.338, "p95_ms": 83.331}, "yolo11_640_fire_11class": {"avg_ms": 49.682, "count": 205, "max_ms": 72.396, "p95_ms": 63.59}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.933, "count": 205, "max_ms": 88.616, "p95_ms": 77.509}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.537, "count": 205, "max_ms": 159.835, "p95_ms": 153.347}, "read_interval": {"avg_ms": 146.681, "count": 204, "max_ms": 171.658, "p95_ms": 165.949}, "triton_wait": {"avg_ms": 64.595, "count": 205, "max_ms": 91.352, "p95_ms": 83.344}, "visualize": {"avg_ms": 4.277, "count": 205, "max_ms": 5.37, "p95_ms": 4.676}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11378}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10884.508, "limit_mb": 8796093022207.996}, "detail": {"pid": 85838, "restart_count": 459, "start_count": 460}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 418, "h264_decode_error_count": 1305, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T06:58:57+09:00", "last_error_summary": "[h264 @ 0x562bd0c97180] error while decoding MB 39 39, bytestream -5", "last_exit_at": "2026-07-01T07:01:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:01:36+09:00", "pid": 85838, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.227, "name": "ffmpeg", "pid": 85838, "rss_mb": 139.227, "state": "S (sleeping)", "threads": 106}, "restart_count": 459, "restart_count_per_hour": 27, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 460}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.137, "sample_count": 1947, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1365377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:01:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.478, "last_frame_pushed_at": "2026-07-01T07:01:34+09:00", "last_frame_read_age_sec": 4.606, "last_frame_read_at": "2026-07-01T07:01:34+09:00", "mjpeg": {"age_sec": 4.459, "exists": true, "mtime": "2026-07-01T07:01:34.387791+09:00", "mtime_epoch": 1782856894.3877914, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186039}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:01:38+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5016.82, "name": "celery", "pid": 267, "rss_mb": 5016.82, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.849, "pushed": 5.849, "read": 5.816}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.18, "count": 175, "max_ms": 49.008, "p95_ms": 34.961}, "gstreamer_push": {"avg_ms": 1.084, "count": 175, "max_ms": 1.244, "p95_ms": 1.154}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.609, "count": 175, "max_ms": 64.646, "p95_ms": 35.221}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.095, "count": 175, "max_ms": 85.929, "p95_ms": 83.523}, "yolo11_640_fire_11class": {"avg_ms": 49.357, "count": 175, "max_ms": 66.313, "p95_ms": 63.476}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.473, "count": 175, "max_ms": 84.117, "p95_ms": 77.904}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.053, "count": 175, "max_ms": 157.354, "p95_ms": 153.056}, "read_interval": {"avg_ms": 146.214, "count": 174, "max_ms": 169.649, "p95_ms": 165.123}, "triton_wait": {"avg_ms": 64.168, "count": 175, "max_ms": 85.942, "p95_ms": 83.536}, "visualize": {"avg_ms": 4.277, "count": 175, "max_ms": 5.37, "p95_ms": 4.676}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10697.488, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c220886680] error while decoding MB 15 19, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 419, "h264_decode_error_count": 1306, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:04:20+09:00", "last_error_summary": "[h264 @ 0x55c220886680] error while decoding MB 15 19, bytestream -5", "last_exit_at": "2026-07-01T07:01:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.582, "last_started_at": "2026-07-01T07:01:36+09:00", "pid": 85838, "process": {"exists": false, "pid": 85838}, "restart_count": 459, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 460}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.392, "sample_count": 1969, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1366467, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:04:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T07:04:20+09:00", "last_frame_read_age_sec": 0.047, "last_frame_read_at": "2026-07-01T07:04:20+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T07:04:20.387849+09:00", "mtime_epoch": 1782857060.3878489, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186519}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:04:20+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5017.078, "name": "celery", "pid": 267, "rss_mb": 5017.078, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.785, "pushed": 6.785, "read": 6.785}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.883, "count": 203, "max_ms": 51.682, "p95_ms": 35.817}, "gstreamer_push": {"avg_ms": 1.081, "count": 203, "max_ms": 1.204, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.828, "count": 203, "max_ms": 48.36, "p95_ms": 35.989}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.915, "count": 203, "max_ms": 90.05, "p95_ms": 86.045}, "yolo11_640_fire_11class": {"avg_ms": 50.893, "count": 203, "max_ms": 69.419, "p95_ms": 64.221}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.449, "count": 203, "max_ms": 82.263, "p95_ms": 78.624}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.653, "count": 203, "max_ms": 174.443, "p95_ms": 154.917}, "read_interval": {"avg_ms": 147.805, "count": 203, "max_ms": 186.533, "p95_ms": 168.289}, "triton_wait": {"avg_ms": 67.619, "count": 203, "max_ms": 90.062, "p95_ms": 86.057}, "visualize": {"avg_ms": 4.189, "count": 203, "max_ms": 5.205, "p95_ms": 4.536}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10696.84, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:04:20+09:00", "last_error_summary": "[h264 @ 0x55c220886680] error while decoding MB 15 19, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 419, "h264_decode_error_count": 1306, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:04:20+09:00", "last_error_summary": "[h264 @ 0x55c220886680] error while decoding MB 15 19, bytestream -5", "last_exit_at": "2026-07-01T07:04:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.667, "last_started_at": "2026-07-01T07:01:36+09:00", "pid": 85838, "process": {"exists": false, "pid": 85838}, "restart_count": 459, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 460}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.392, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1366468, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:04:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:04:20+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-01T07:04:20+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-01T07:04:20.387849+09:00", "mtime_epoch": 1782857060.3878489, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186519}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:04:20+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5017.078, "name": "celery", "pid": 267, "rss_mb": 5017.078, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.769, "pushed": 6.769, "read": 6.769}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.889, "count": 203, "max_ms": 51.682, "p95_ms": 35.817}, "gstreamer_push": {"avg_ms": 1.081, "count": 203, "max_ms": 1.204, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.827, "count": 203, "max_ms": 48.36, "p95_ms": 35.989}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.873, "count": 203, "max_ms": 90.05, "p95_ms": 86.045}, "yolo11_640_fire_11class": {"avg_ms": 50.862, "count": 203, "max_ms": 69.419, "p95_ms": 64.221}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.438, "count": 203, "max_ms": 82.263, "p95_ms": 78.624}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.626, "count": 203, "max_ms": 174.443, "p95_ms": 154.917}, "read_interval": {"avg_ms": 147.805, "count": 203, "max_ms": 186.533, "p95_ms": 168.289}, "triton_wait": {"avg_ms": 67.577, "count": 203, "max_ms": 90.062, "p95_ms": 86.057}, "visualize": {"avg_ms": 4.191, "count": 203, "max_ms": 5.205, "p95_ms": 4.536}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10696.816, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 419, "h264_decode_error_count": 1306, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:04:20+09:00", "last_error_summary": "[h264 @ 0x55c220886680] error while decoding MB 15 19, bytestream -5", "last_exit_at": "2026-07-01T07:04:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.672, "last_started_at": "2026-07-01T07:01:36+09:00", "pid": 85838, "process": {"exists": false, "pid": 85838}, "restart_count": 459, "restart_count_per_hour": 26, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 460}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.392, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1366468, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:04:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:04:20+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-01T07:04:20+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T07:04:20.387849+09:00", "mtime_epoch": 1782857060.3878489, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186519}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:04:20+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5017.078, "name": "celery", "pid": 267, "rss_mb": 5017.078, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.767, "pushed": 6.767, "read": 6.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.889, "count": 203, "max_ms": 51.682, "p95_ms": 35.817}, "gstreamer_push": {"avg_ms": 1.081, "count": 203, "max_ms": 1.204, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.827, "count": 203, "max_ms": 48.36, "p95_ms": 35.989}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.873, "count": 203, "max_ms": 90.05, "p95_ms": 86.045}, "yolo11_640_fire_11class": {"avg_ms": 50.862, "count": 203, "max_ms": 69.419, "p95_ms": 64.221}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.438, "count": 203, "max_ms": 82.263, "p95_ms": 78.624}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.626, "count": 203, "max_ms": 174.443, "p95_ms": 154.917}, "read_interval": {"avg_ms": 147.805, "count": 203, "max_ms": 186.533, "p95_ms": 168.289}, "triton_wait": {"avg_ms": 67.577, "count": 203, "max_ms": 90.062, "p95_ms": 86.057}, "visualize": {"avg_ms": 4.191, "count": 203, "max_ms": 5.205, "p95_ms": 4.536}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11387}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10805.172, "limit_mb": 8796093022207.996}, "detail": {"pid": 86005, "restart_count": 460, "start_count": 461}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 419, "h264_decode_error_count": 1306, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:04:20+09:00", "last_error_summary": "[h264 @ 0x55c220886680] error while decoding MB 15 19, bytestream -5", "last_exit_at": "2026-07-01T07:04:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:04:23+09:00", "pid": 86005, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.625, "name": "ffmpeg", "pid": 86005, "rss_mb": 139.625, "state": "S (sleeping)", "threads": 106}, "restart_count": 460, "restart_count_per_hour": 27, "rtsp_timeout_count": 388, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 461}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.437, "sample_count": 1939, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1366468, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:04:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.495, "last_frame_pushed_at": "2026-07-01T07:04:20+09:00", "last_frame_read_age_sec": 4.622, "last_frame_read_at": "2026-07-01T07:04:20+09:00", "mjpeg": {"age_sec": 4.474, "exists": true, "mtime": "2026-07-01T07:04:20.527852+09:00", "mtime_epoch": 1782857060.527852, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184013}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:04:25+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5017.078, "name": "celery", "pid": 267, "rss_mb": 5017.078, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.808, "pushed": 5.808, "read": 5.775}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.928, "count": 174, "max_ms": 51.682, "p95_ms": 35.88}, "gstreamer_push": {"avg_ms": 1.079, "count": 174, "max_ms": 1.204, "p95_ms": 1.162}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.981, "count": 174, "max_ms": 48.36, "p95_ms": 36.559}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.327, "count": 174, "max_ms": 89.932, "p95_ms": 86.045}, "yolo11_640_fire_11class": {"avg_ms": 50.871, "count": 174, "max_ms": 69.419, "p95_ms": 64.282}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.391, "count": 174, "max_ms": 82.263, "p95_ms": 78.624}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.005, "count": 174, "max_ms": 174.443, "p95_ms": 154.994}, "read_interval": {"avg_ms": 147.213, "count": 173, "max_ms": 186.533, "p95_ms": 168.375}, "triton_wait": {"avg_ms": 67.146, "count": 174, "max_ms": 89.944, "p95_ms": 86.057}, "visualize": {"avg_ms": 4.19, "count": 174, "max_ms": 5.205, "p95_ms": 4.585}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11393}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10698.461, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c9c8d15a40] cabac decode of qscale diff failed at 18 34 [h264 @ 0x55c9c8d15a40] error while decoding MB 18 34, bytestream 4714 [h264 @ 0x55c9c97ecd40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c9c97ecd40] error while decoding MB 0 30, bytestream 123464 [h264 @ 0x55c9c8ca8fc0] reference picture missing during reorder [h264 @ 0x55c9c8ca8fc0] Missing reference picture, default is 65581 [h264 @ 0x55c9c8cbd0c0] mmco: unref short failure [h264 @ 0x55c9c8cbd0c0] ille", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 420, "h264_decode_error_count": 1311, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:06:43+09:00", "last_error_summary": "[h264 @ 0x55c9c8d15a40] cabac decode of qscale diff failed at 18 34 [h264 @ 0x55c9c8d15a40] error while decoding MB 18 34, bytestream 4714 [h264 @ 0x55c9c97ecd40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c9c97ecd40] error while decoding MB 0 30, bytestream 123464 [h264 @ 0x55c9c8ca8fc0] reference picture missing during reorder [h264 @ 0x55c9c8ca8fc0] Missing reference picture, default is 65581 [h264 @ 0x55c9c8cbd0c0] mmco: unref short failure [h264 @ 0x55c9c8cbd0c0] ille", "last_exit_at": "2026-07-01T07:04:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.27, "last_started_at": "2026-07-01T07:04:23+09:00", "pid": 86005, "process": {"exists": false, "pid": 86005}, "restart_count": 460, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 461}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.912, "sample_count": 1864, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1367272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:06:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T07:06:43+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T07:06:43+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T07:06:43.227047+09:00", "mtime_epoch": 1782857203.2270474, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197536}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:06:43+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5017.594, "name": "celery", "pid": 267, "rss_mb": 5017.594, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 75, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.503, "pushed": 2.503, "read": 2.503}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.987, "count": 74, "max_ms": 36.876, "p95_ms": 35.562}, "gstreamer_push": {"avg_ms": 1.102, "count": 75, "max_ms": 1.187, "p95_ms": 1.164}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.74, "count": 74, "max_ms": 37.74, "p95_ms": 33.787}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.185, "count": 74, "max_ms": 88.769, "p95_ms": 85.359}, "yolo11_640_fire_11class": {"avg_ms": 53.117, "count": 74, "max_ms": 71.533, "p95_ms": 68.967}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.439, "count": 74, "max_ms": 79.651, "p95_ms": 78.234}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.186, "count": 75, "max_ms": 160.298, "p95_ms": 154.845}, "read_interval": {"avg_ms": 400.637, "count": 75, "max_ms": 10133.051, "p95_ms": 168.411}, "triton_wait": {"avg_ms": 67.439, "count": 74, "max_ms": 88.781, "p95_ms": 85.372}, "visualize": {"avg_ms": 4.312, "count": 74, "max_ms": 5.145, "p95_ms": 4.584}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11393}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10698.207, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:06:43+09:00", "last_error_summary": "[h264 @ 0x55c9c8d15a40] cabac decode of qscale diff failed at 18 34 [h264 @ 0x55c9c8d15a40] error while decoding MB 18 34, bytestream 4714 [h264 @ 0x55c9c97ecd40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c9c97ecd40] error while decoding MB 0 30, bytestream 123464 [h264 @ 0x55c9c8ca8fc0] reference picture missing during reorder [h264 @ 0x55c9c8ca8fc0] Missing reference picture, default is 65581 [h264 @ 0x55c9c8cbd0c0] mmco: unref short failure [h264 @ 0x55c9c8cbd0c0] ille", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 420, "h264_decode_error_count": 1311, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:06:43+09:00", "last_error_summary": "[h264 @ 0x55c9c8d15a40] cabac decode of qscale diff failed at 18 34 [h264 @ 0x55c9c8d15a40] error while decoding MB 18 34, bytestream 4714 [h264 @ 0x55c9c97ecd40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c9c97ecd40] error while decoding MB 0 30, bytestream 123464 [h264 @ 0x55c9c8ca8fc0] reference picture missing during reorder [h264 @ 0x55c9c8ca8fc0] Missing reference picture, default is 65581 [h264 @ 0x55c9c8cbd0c0] mmco: unref short failure [h264 @ 0x55c9c8cbd0c0] ille", "last_exit_at": "2026-07-01T07:06:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.359, "last_started_at": "2026-07-01T07:04:23+09:00", "pid": 86005, "process": {"exists": false, "pid": 86005}, "restart_count": 460, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 461}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.912, "sample_count": 1865, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1367273, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:06:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T07:06:43+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-01T07:06:43+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-01T07:06:43.227047+09:00", "mtime_epoch": 1782857203.2270474, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197536}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:06:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5017.594, "name": "celery", "pid": 267, "rss_mb": 5017.594, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 74, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.507, "pushed": 2.507, "read": 2.474}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.03, "count": 75, "max_ms": 36.876, "p95_ms": 35.562}, "gstreamer_push": {"avg_ms": 1.103, "count": 75, "max_ms": 1.187, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.77, "count": 75, "max_ms": 37.74, "p95_ms": 33.787}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.064, "count": 75, "max_ms": 88.769, "p95_ms": 85.359}, "yolo11_640_fire_11class": {"avg_ms": 53.022, "count": 75, "max_ms": 71.533, "p95_ms": 68.967}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.352, "count": 75, "max_ms": 79.651, "p95_ms": 78.234}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.229, "count": 75, "max_ms": 160.298, "p95_ms": 154.845}, "read_interval": {"avg_ms": 404.077, "count": 74, "max_ms": 10133.051, "p95_ms": 168.411}, "triton_wait": {"avg_ms": 67.315, "count": 75, "max_ms": 88.781, "p95_ms": 85.372}, "visualize": {"avg_ms": 4.323, "count": 75, "max_ms": 5.162, "p95_ms": 4.966}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11393}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10698.199, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 420, "h264_decode_error_count": 1311, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:06:43+09:00", "last_error_summary": "[h264 @ 0x55c9c8d15a40] cabac decode of qscale diff failed at 18 34 [h264 @ 0x55c9c8d15a40] error while decoding MB 18 34, bytestream 4714 [h264 @ 0x55c9c97ecd40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c9c97ecd40] error while decoding MB 0 30, bytestream 123464 [h264 @ 0x55c9c8ca8fc0] reference picture missing during reorder [h264 @ 0x55c9c8ca8fc0] Missing reference picture, default is 65581 [h264 @ 0x55c9c8cbd0c0] mmco: unref short failure [h264 @ 0x55c9c8cbd0c0] ille", "last_exit_at": "2026-07-01T07:06:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.364, "last_started_at": "2026-07-01T07:04:23+09:00", "pid": 86005, "process": {"exists": false, "pid": 86005}, "restart_count": 460, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 461}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.912, "sample_count": 1865, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1367273, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:06:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T07:06:43+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-01T07:06:43+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-01T07:06:43.227047+09:00", "mtime_epoch": 1782857203.2270474, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197536}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:06:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5017.594, "name": "celery", "pid": 267, "rss_mb": 5017.594, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 74, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.507, "pushed": 2.507, "read": 2.473}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.03, "count": 75, "max_ms": 36.876, "p95_ms": 35.562}, "gstreamer_push": {"avg_ms": 1.103, "count": 75, "max_ms": 1.187, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.77, "count": 75, "max_ms": 37.74, "p95_ms": 33.787}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.064, "count": 75, "max_ms": 88.769, "p95_ms": 85.359}, "yolo11_640_fire_11class": {"avg_ms": 53.022, "count": 75, "max_ms": 71.533, "p95_ms": 68.967}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.352, "count": 75, "max_ms": 79.651, "p95_ms": 78.234}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.229, "count": 75, "max_ms": 160.298, "p95_ms": 154.845}, "read_interval": {"avg_ms": 404.077, "count": 74, "max_ms": 10133.051, "p95_ms": 168.411}, "triton_wait": {"avg_ms": 67.315, "count": 75, "max_ms": 88.781, "p95_ms": 85.372}, "visualize": {"avg_ms": 4.323, "count": 75, "max_ms": 5.162, "p95_ms": 4.966}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11393}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10806.711, "limit_mb": 8796093022207.996}, "detail": {"pid": 86112, "restart_count": 461, "start_count": 462}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 420, "h264_decode_error_count": 1311, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:06:43+09:00", "last_error_summary": "[h264 @ 0x55c9c8d15a40] cabac decode of qscale diff failed at 18 34 [h264 @ 0x55c9c8d15a40] error while decoding MB 18 34, bytestream 4714 [h264 @ 0x55c9c97ecd40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c9c97ecd40] error while decoding MB 0 30, bytestream 123464 [h264 @ 0x55c9c8ca8fc0] reference picture missing during reorder [h264 @ 0x55c9c8ca8fc0] Missing reference picture, default is 65581 [h264 @ 0x55c9c8cbd0c0] mmco: unref short failure [h264 @ 0x55c9c8cbd0c0] ille", "last_exit_at": "2026-07-01T07:06:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:06:45+09:00", "pid": 86112, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.934, "name": "ffmpeg", "pid": 86112, "rss_mb": 138.934, "state": "S (sleeping)", "threads": 106}, "restart_count": 461, "restart_count_per_hour": 26, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 462}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.912, "sample_count": 1836, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1367273, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:06:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.101, "last_frame_pushed_at": "2026-07-01T07:06:43+09:00", "last_frame_read_age_sec": 4.241, "last_frame_read_at": "2026-07-01T07:06:43+09:00", "mjpeg": {"age_sec": 4.082, "exists": true, "mtime": "2026-07-01T07:06:43.375051+09:00", "mtime_epoch": 1782857203.3750505, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198321}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:06:47+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5017.594, "name": "celery", "pid": 267, "rss_mb": 5017.594, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 48, "pushed_frames": 48, "read_frames": 47, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.6, "pushed": 1.6, "read": 1.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.885, "count": 47, "max_ms": 36.273, "p95_ms": 34.975}, "gstreamer_push": {"avg_ms": 1.103, "count": 48, "max_ms": 1.187, "p95_ms": 1.177}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.173, "count": 47, "max_ms": 37.051, "p95_ms": 33.787}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.536, "count": 47, "max_ms": 88.769, "p95_ms": 85.761}, "yolo11_640_fire_11class": {"avg_ms": 53.021, "count": 47, "max_ms": 71.533, "p95_ms": 66.994}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.984, "count": 47, "max_ms": 79.651, "p95_ms": 78.234}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.517, "count": 48, "max_ms": 160.298, "p95_ms": 155.79}, "read_interval": {"avg_ms": 550.779, "count": 47, "max_ms": 10133.051, "p95_ms": 172.352}, "triton_wait": {"avg_ms": 67.929, "count": 47, "max_ms": 88.781, "p95_ms": 85.773}, "visualize": {"avg_ms": 4.305, "count": 47, "max_ms": 5.162, "p95_ms": 4.55}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10700.711, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56500f7ea1c0] error while decoding MB 12 9, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 421, "h264_decode_error_count": 1312, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:09:28+09:00", "last_error_summary": "[h264 @ 0x56500f7ea1c0] error while decoding MB 12 9, bytestream -9", "last_exit_at": "2026-07-01T07:06:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.513, "last_started_at": "2026-07-01T07:06:45+09:00", "pid": 86112, "process": {"exists": false, "pid": 86112}, "restart_count": 461, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 462}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.385, "sample_count": 1881, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1368376, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:09:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T07:09:28+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T07:09:28+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T07:09:28.922509+09:00", "mtime_epoch": 1782857368.922509, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192988}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:09:29+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5018.109, "name": "celery", "pid": 267, "rss_mb": 5018.109, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.918, "pushed": 6.918, "read": 6.918}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.343, "count": 207, "max_ms": 50.648, "p95_ms": 35.489}, "gstreamer_push": {"avg_ms": 1.099, "count": 207, "max_ms": 1.646, "p95_ms": 1.163}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.623, "count": 207, "max_ms": 42.31, "p95_ms": 34.16}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.082, "count": 207, "max_ms": 87.14, "p95_ms": 80.914}, "yolo11_640_fire_11class": {"avg_ms": 49.84, "count": 207, "max_ms": 68.075, "p95_ms": 62.904}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.527, "count": 207, "max_ms": 80.745, "p95_ms": 77.385}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.416, "count": 207, "max_ms": 159.058, "p95_ms": 151.513}, "read_interval": {"avg_ms": 144.938, "count": 207, "max_ms": 172.028, "p95_ms": 163.316}, "triton_wait": {"avg_ms": 63.184, "count": 207, "max_ms": 87.151, "p95_ms": 80.926}, "visualize": {"avg_ms": 4.234, "count": 207, "max_ms": 5.071, "p95_ms": 4.645}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10700.234, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:09:28+09:00", "last_error_summary": "[h264 @ 0x56500f7ea1c0] error while decoding MB 12 9, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 421, "h264_decode_error_count": 1312, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:09:28+09:00", "last_error_summary": "[h264 @ 0x56500f7ea1c0] error while decoding MB 12 9, bytestream -9", "last_exit_at": "2026-07-01T07:09:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.608, "last_started_at": "2026-07-01T07:06:45+09:00", "pid": 86112, "process": {"exists": false, "pid": 86112}, "restart_count": 461, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 462}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.385, "sample_count": 1881, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1368377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:09:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:09:29+09:00", "last_frame_read_age_sec": 0.147, "last_frame_read_at": "2026-07-01T07:09:28+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-01T07:09:28.922509+09:00", "mtime_epoch": 1782857368.922509, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192988}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:09:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5018.109, "name": "celery", "pid": 267, "rss_mb": 5018.109, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.926, "pushed": 6.926, "read": 6.893}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.365, "count": 207, "max_ms": 50.648, "p95_ms": 35.542}, "gstreamer_push": {"avg_ms": 1.099, "count": 207, "max_ms": 1.646, "p95_ms": 1.163}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.6, "count": 207, "max_ms": 42.31, "p95_ms": 34.15}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.071, "count": 207, "max_ms": 87.14, "p95_ms": 80.914}, "yolo11_640_fire_11class": {"avg_ms": 49.835, "count": 207, "max_ms": 68.075, "p95_ms": 62.904}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.539, "count": 207, "max_ms": 80.745, "p95_ms": 77.385}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.462, "count": 207, "max_ms": 159.058, "p95_ms": 151.513}, "read_interval": {"avg_ms": 144.964, "count": 206, "max_ms": 172.028, "p95_ms": 163.316}, "triton_wait": {"avg_ms": 63.173, "count": 207, "max_ms": 87.151, "p95_ms": 80.926}, "visualize": {"avg_ms": 4.237, "count": 207, "max_ms": 5.071, "p95_ms": 4.707}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.886, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10700.465, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 421, "h264_decode_error_count": 1312, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:09:28+09:00", "last_error_summary": "[h264 @ 0x56500f7ea1c0] error while decoding MB 12 9, bytestream -9", "last_exit_at": "2026-07-01T07:09:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.614, "last_started_at": "2026-07-01T07:06:45+09:00", "pid": 86112, "process": {"exists": false, "pid": 86112}, "restart_count": 461, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 462}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.385, "sample_count": 1881, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1368377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:09:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T07:09:29+09:00", "last_frame_read_age_sec": 0.153, "last_frame_read_at": "2026-07-01T07:09:28+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-01T07:09:28.922509+09:00", "mtime_epoch": 1782857368.922509, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192988}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:09:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5018.109, "name": "celery", "pid": 267, "rss_mb": 5018.109, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.925, "pushed": 6.925, "read": 6.892}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.365, "count": 207, "max_ms": 50.648, "p95_ms": 35.542}, "gstreamer_push": {"avg_ms": 1.099, "count": 207, "max_ms": 1.646, "p95_ms": 1.163}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.6, "count": 207, "max_ms": 42.31, "p95_ms": 34.15}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.071, "count": 207, "max_ms": 87.14, "p95_ms": 80.914}, "yolo11_640_fire_11class": {"avg_ms": 49.835, "count": 207, "max_ms": 68.075, "p95_ms": 62.904}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.539, "count": 207, "max_ms": 80.745, "p95_ms": 77.385}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.462, "count": 207, "max_ms": 159.058, "p95_ms": 151.513}, "read_interval": {"avg_ms": 144.964, "count": 206, "max_ms": 172.028, "p95_ms": 163.316}, "triton_wait": {"avg_ms": 63.173, "count": 207, "max_ms": 87.151, "p95_ms": 80.926}, "visualize": {"avg_ms": 4.237, "count": 207, "max_ms": 5.071, "p95_ms": 4.707}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10805.887, "limit_mb": 8796093022207.996}, "detail": {"pid": 86219, "restart_count": 462, "start_count": 463}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 421, "h264_decode_error_count": 1312, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:09:28+09:00", "last_error_summary": "[h264 @ 0x56500f7ea1c0] error while decoding MB 12 9, bytestream -9", "last_exit_at": "2026-07-01T07:09:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:09:31+09:00", "pid": 86219, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.035, "name": "ffmpeg", "pid": 86219, "rss_mb": 136.035, "state": "S (sleeping)", "threads": 106}, "restart_count": 462, "restart_count_per_hour": 26, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 463}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.424, "sample_count": 1851, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1368377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:09:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.486, "last_frame_pushed_at": "2026-07-01T07:09:29+09:00", "last_frame_read_age_sec": 4.627, "last_frame_read_at": "2026-07-01T07:09:28+09:00", "mjpeg": {"age_sec": 4.47, "exists": true, "mtime": "2026-07-01T07:09:29.074512+09:00", "mtime_epoch": 1782857369.074512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193174}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:09:33+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5018.109, "name": "celery", "pid": 267, "rss_mb": 5018.109, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 176, "pushed_frames": 176, "read_frames": 175, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.873, "pushed": 5.873, "read": 5.839}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.39, "count": 175, "max_ms": 50.648, "p95_ms": 36.076}, "gstreamer_push": {"avg_ms": 1.099, "count": 176, "max_ms": 1.646, "p95_ms": 1.168}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.399, "count": 175, "max_ms": 40.833, "p95_ms": 33.928}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.446, "count": 175, "max_ms": 87.14, "p95_ms": 80.982}, "yolo11_640_fire_11class": {"avg_ms": 50.096, "count": 175, "max_ms": 68.075, "p95_ms": 62.904}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.986, "count": 175, "max_ms": 80.745, "p95_ms": 77.585}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.878, "count": 176, "max_ms": 159.058, "p95_ms": 152.118}, "read_interval": {"avg_ms": 145.572, "count": 175, "max_ms": 172.028, "p95_ms": 164.045}, "triton_wait": {"avg_ms": 63.558, "count": 175, "max_ms": 87.151, "p95_ms": 80.995}, "visualize": {"avg_ms": 4.243, "count": 175, "max_ms": 5.071, "p95_ms": 4.743}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11412}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10702.641, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5583de743100] error while decoding MB 5 19, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 422, "h264_decode_error_count": 1313, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:12:15+09:00", "last_error_summary": "[h264 @ 0x5583de743100] error while decoding MB 5 19, bytestream -11", "last_exit_at": "2026-07-01T07:09:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.613, "last_started_at": "2026-07-01T07:09:31+09:00", "pid": 86219, "process": {"exists": false, "pid": 86219}, "restart_count": 462, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 463}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.484, "sample_count": 2026, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1369487, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:12:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-01T07:12:15+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-01T07:12:15+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-01T07:12:15.105783+09:00", "mtime_epoch": 1782857535.1057832, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191344}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:12:15+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5018.625, "name": "celery", "pid": 267, "rss_mb": 5018.625, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.829, "pushed": 6.829, "read": 6.829}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.18, "count": 204, "max_ms": 42.662, "p95_ms": 35.239}, "gstreamer_push": {"avg_ms": 1.097, "count": 204, "max_ms": 1.205, "p95_ms": 1.172}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.586, "count": 204, "max_ms": 37.56, "p95_ms": 34.706}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.687, "count": 204, "max_ms": 85.057, "p95_ms": 82.551}, "yolo11_640_fire_11class": {"avg_ms": 49.798, "count": 204, "max_ms": 66.023, "p95_ms": 62.816}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.974, "count": 204, "max_ms": 83.677, "p95_ms": 78.179}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.269, "count": 204, "max_ms": 156.613, "p95_ms": 152.316}, "read_interval": {"avg_ms": 146.856, "count": 204, "max_ms": 169.273, "p95_ms": 164.266}, "triton_wait": {"avg_ms": 65.028, "count": 204, "max_ms": 85.07, "p95_ms": 82.563}, "visualize": {"avg_ms": 4.271, "count": 204, "max_ms": 4.856, "p95_ms": 4.545}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11412}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10702.035, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:12:15+09:00", "last_error_summary": "[h264 @ 0x5583de743100] error while decoding MB 5 19, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 422, "h264_decode_error_count": 1313, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:12:15+09:00", "last_error_summary": "[h264 @ 0x5583de743100] error while decoding MB 5 19, bytestream -11", "last_exit_at": "2026-07-01T07:12:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.699, "last_started_at": "2026-07-01T07:09:31+09:00", "pid": 86219, "process": {"exists": false, "pid": 86219}, "restart_count": 462, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 463}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.484, "sample_count": 2027, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1369488, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:12:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T07:12:15+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-01T07:12:15+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-01T07:12:15.105783+09:00", "mtime_epoch": 1782857535.1057832, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191344}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:12:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5018.625, "name": "celery", "pid": 267, "rss_mb": 5018.625, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.843, "pushed": 6.843, "read": 6.809}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.182, "count": 205, "max_ms": 42.662, "p95_ms": 35.239}, "gstreamer_push": {"avg_ms": 1.097, "count": 205, "max_ms": 1.205, "p95_ms": 1.172}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.583, "count": 205, "max_ms": 37.56, "p95_ms": 34.706}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.69, "count": 205, "max_ms": 85.057, "p95_ms": 82.551}, "yolo11_640_fire_11class": {"avg_ms": 49.754, "count": 205, "max_ms": 66.023, "p95_ms": 62.816}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.01, "count": 205, "max_ms": 83.677, "p95_ms": 78.179}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.277, "count": 205, "max_ms": 156.613, "p95_ms": 152.316}, "read_interval": {"avg_ms": 146.856, "count": 204, "max_ms": 169.273, "p95_ms": 164.266}, "triton_wait": {"avg_ms": 65.034, "count": 205, "max_ms": 85.07, "p95_ms": 82.563}, "visualize": {"avg_ms": 4.271, "count": 205, "max_ms": 4.856, "p95_ms": 4.545}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11412}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10701.98, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 422, "h264_decode_error_count": 1313, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:12:15+09:00", "last_error_summary": "[h264 @ 0x5583de743100] error while decoding MB 5 19, bytestream -11", "last_exit_at": "2026-07-01T07:12:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.704, "last_started_at": "2026-07-01T07:09:31+09:00", "pid": 86219, "process": {"exists": false, "pid": 86219}, "restart_count": 462, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 463}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.484, "sample_count": 2027, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1369488, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:12:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:12:15+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-01T07:12:15+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-01T07:12:15.105783+09:00", "mtime_epoch": 1782857535.1057832, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191344}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:12:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5018.625, "name": "celery", "pid": 267, "rss_mb": 5018.625, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.841, "pushed": 6.841, "read": 6.808}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.182, "count": 205, "max_ms": 42.662, "p95_ms": 35.239}, "gstreamer_push": {"avg_ms": 1.097, "count": 205, "max_ms": 1.205, "p95_ms": 1.172}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.583, "count": 205, "max_ms": 37.56, "p95_ms": 34.706}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.69, "count": 205, "max_ms": 85.057, "p95_ms": 82.551}, "yolo11_640_fire_11class": {"avg_ms": 49.754, "count": 205, "max_ms": 66.023, "p95_ms": 62.816}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.01, "count": 205, "max_ms": 83.677, "p95_ms": 78.179}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.277, "count": 205, "max_ms": 156.613, "p95_ms": 152.316}, "read_interval": {"avg_ms": 146.856, "count": 204, "max_ms": 169.273, "p95_ms": 164.266}, "triton_wait": {"avg_ms": 65.034, "count": 205, "max_ms": 85.07, "p95_ms": 82.563}, "visualize": {"avg_ms": 4.271, "count": 205, "max_ms": 4.856, "p95_ms": 4.545}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11412}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10810.438, "limit_mb": 8796093022207.996}, "detail": {"pid": 86326, "restart_count": 463, "start_count": 464}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 422, "h264_decode_error_count": 1313, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:12:15+09:00", "last_error_summary": "[h264 @ 0x5583de743100] error while decoding MB 5 19, bytestream -11", "last_exit_at": "2026-07-01T07:12:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:12:17+09:00", "pid": 86326, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.809, "name": "ffmpeg", "pid": 86326, "rss_mb": 139.809, "state": "S (sleeping)", "threads": 106}, "restart_count": 463, "restart_count_per_hour": 26, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 464}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.458, "sample_count": 1997, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1369488, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:12:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.404, "last_frame_pushed_at": "2026-07-01T07:12:15+09:00", "last_frame_read_age_sec": 4.542, "last_frame_read_at": "2026-07-01T07:12:15+09:00", "mjpeg": {"age_sec": 4.388, "exists": true, "mtime": "2026-07-01T07:12:15.253786+09:00", "mtime_epoch": 1782857535.253786, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:12:19+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5018.625, "name": "celery", "pid": 267, "rss_mb": 5018.625, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.8, "pushed": 5.833, "read": 5.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.181, "count": 174, "max_ms": 35.634, "p95_ms": 35.205}, "gstreamer_push": {"avg_ms": 1.097, "count": 175, "max_ms": 1.205, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.676, "count": 174, "max_ms": 37.56, "p95_ms": 35.162}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.94, "count": 174, "max_ms": 85.057, "p95_ms": 82.843}, "yolo11_640_fire_11class": {"avg_ms": 49.644, "count": 174, "max_ms": 66.023, "p95_ms": 62.776}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.984, "count": 174, "max_ms": 83.677, "p95_ms": 78.597}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.488, "count": 174, "max_ms": 156.613, "p95_ms": 152.791}, "read_interval": {"avg_ms": 147.033, "count": 174, "max_ms": 169.273, "p95_ms": 164.65}, "triton_wait": {"avg_ms": 65.189, "count": 174, "max_ms": 85.07, "p95_ms": 82.852}, "visualize": {"avg_ms": 4.278, "count": 174, "max_ms": 4.856, "p95_ms": 4.545}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11421}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10704.066, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5563fe7e6d00] error while decoding MB 31 19, bytestream -32", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 423, "h264_decode_error_count": 1314, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:15:00+09:00", "last_error_summary": "[h264 @ 0x5563fe7e6d00] error while decoding MB 31 19, bytestream -32", "last_exit_at": "2026-07-01T07:12:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.33, "last_started_at": "2026-07-01T07:12:17+09:00", "pid": 86326, "process": {"exists": false, "pid": 86326}, "restart_count": 463, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 464}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.658, "sample_count": 2026, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1370590, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:15:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T07:15:00+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T07:15:00+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T07:15:00.924908+09:00", "mtime_epoch": 1782857700.9249082, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190156}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:15:00+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5019.141, "name": "celery", "pid": 267, "rss_mb": 5019.141, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.838, "pushed": 6.838, "read": 6.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.941, "count": 204, "max_ms": 48.75, "p95_ms": 34.876}, "gstreamer_push": {"avg_ms": 1.086, "count": 205, "max_ms": 1.277, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.723, "count": 204, "max_ms": 41.647, "p95_ms": 33.693}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.528, "count": 204, "max_ms": 89.444, "p95_ms": 84.676}, "yolo11_640_fire_11class": {"avg_ms": 49.736, "count": 204, "max_ms": 65.346, "p95_ms": 63.725}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.986, "count": 204, "max_ms": 82.522, "p95_ms": 78.3}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.455, "count": 205, "max_ms": 160.944, "p95_ms": 154.416}, "read_interval": {"avg_ms": 146.582, "count": 205, "max_ms": 172.85, "p95_ms": 166.468}, "triton_wait": {"avg_ms": 64.711, "count": 204, "max_ms": 89.456, "p95_ms": 84.689}, "visualize": {"avg_ms": 4.221, "count": 204, "max_ms": 4.634, "p95_ms": 4.472}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11421}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10703.691, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:15:00+09:00", "last_error_summary": "[h264 @ 0x5563fe7e6d00] error while decoding MB 31 19, bytestream -32", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 423, "h264_decode_error_count": 1314, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:15:00+09:00", "last_error_summary": "[h264 @ 0x5563fe7e6d00] error while decoding MB 31 19, bytestream -32", "last_exit_at": "2026-07-01T07:15:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.418, "last_started_at": "2026-07-01T07:12:17+09:00", "pid": 86326, "process": {"exists": false, "pid": 86326}, "restart_count": 463, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 464}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.658, "sample_count": 2026, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1370591, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:15:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T07:15:01+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-01T07:15:00+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-01T07:15:00.924908+09:00", "mtime_epoch": 1782857700.9249082, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190156}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:15:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5019.141, "name": "celery", "pid": 267, "rss_mb": 5019.141, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.85, "pushed": 6.85, "read": 6.817}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.953, "count": 205, "max_ms": 48.75, "p95_ms": 34.876}, "gstreamer_push": {"avg_ms": 1.087, "count": 205, "max_ms": 1.277, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.713, "count": 205, "max_ms": 41.647, "p95_ms": 33.693}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.515, "count": 205, "max_ms": 89.444, "p95_ms": 84.676}, "yolo11_640_fire_11class": {"avg_ms": 49.761, "count": 205, "max_ms": 65.346, "p95_ms": 63.725}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.971, "count": 205, "max_ms": 82.522, "p95_ms": 78.3}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.51, "count": 205, "max_ms": 160.944, "p95_ms": 154.416}, "read_interval": {"avg_ms": 146.65, "count": 204, "max_ms": 172.85, "p95_ms": 166.468}, "triton_wait": {"avg_ms": 64.696, "count": 205, "max_ms": 89.456, "p95_ms": 84.689}, "visualize": {"avg_ms": 4.22, "count": 205, "max_ms": 4.634, "p95_ms": 4.472}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11421}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10703.676, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 423, "h264_decode_error_count": 1314, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:15:00+09:00", "last_error_summary": "[h264 @ 0x5563fe7e6d00] error while decoding MB 31 19, bytestream -32", "last_exit_at": "2026-07-01T07:15:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.423, "last_started_at": "2026-07-01T07:12:17+09:00", "pid": 86326, "process": {"exists": false, "pid": 86326}, "restart_count": 463, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 464}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.658, "sample_count": 2026, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1370591, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:15:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:15:01+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-01T07:15:00+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-01T07:15:00.924908+09:00", "mtime_epoch": 1782857700.9249082, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190156}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:15:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5019.141, "name": "celery", "pid": 267, "rss_mb": 5019.141, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.849, "pushed": 6.849, "read": 6.815}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.953, "count": 205, "max_ms": 48.75, "p95_ms": 34.876}, "gstreamer_push": {"avg_ms": 1.087, "count": 205, "max_ms": 1.277, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.713, "count": 205, "max_ms": 41.647, "p95_ms": 33.693}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.515, "count": 205, "max_ms": 89.444, "p95_ms": 84.676}, "yolo11_640_fire_11class": {"avg_ms": 49.761, "count": 205, "max_ms": 65.346, "p95_ms": 63.725}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.971, "count": 205, "max_ms": 82.522, "p95_ms": 78.3}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.51, "count": 205, "max_ms": 160.944, "p95_ms": 154.416}, "read_interval": {"avg_ms": 146.65, "count": 204, "max_ms": 172.85, "p95_ms": 166.468}, "triton_wait": {"avg_ms": 64.696, "count": 205, "max_ms": 89.456, "p95_ms": 84.689}, "visualize": {"avg_ms": 4.22, "count": 205, "max_ms": 4.634, "p95_ms": 4.472}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11421}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10808.824, "limit_mb": 8796093022207.996}, "detail": {"pid": 86433, "restart_count": 464, "start_count": 465}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 423, "h264_decode_error_count": 1314, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:15:00+09:00", "last_error_summary": "[h264 @ 0x5563fe7e6d00] error while decoding MB 31 19, bytestream -32", "last_exit_at": "2026-07-01T07:15:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:15:03+09:00", "pid": 86433, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.695, "name": "ffmpeg", "pid": 86433, "rss_mb": 136.695, "state": "S (sleeping)", "threads": 106}, "restart_count": 464, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 465}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.503, "sample_count": 1995, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1370591, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:15:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.619, "last_frame_pushed_at": "2026-07-01T07:15:01+09:00", "last_frame_read_age_sec": 4.753, "last_frame_read_at": "2026-07-01T07:15:00+09:00", "mjpeg": {"age_sec": 4.605, "exists": true, "mtime": "2026-07-01T07:15:01.068911+09:00", "mtime_epoch": 1782857701.0689108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190273}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:15:05+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5019.141, "name": "celery", "pid": 267, "rss_mb": 5019.141, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 172, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.734, "pushed": 5.734, "read": 5.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.86, "count": 172, "max_ms": 48.75, "p95_ms": 34.269}, "gstreamer_push": {"avg_ms": 1.091, "count": 172, "max_ms": 1.277, "p95_ms": 1.147}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.493, "count": 172, "max_ms": 41.647, "p95_ms": 33.434}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.431, "count": 172, "max_ms": 89.444, "p95_ms": 84.941}, "yolo11_640_fire_11class": {"avg_ms": 50.331, "count": 172, "max_ms": 65.346, "p95_ms": 63.792}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.695, "count": 172, "max_ms": 82.522, "p95_ms": 78.548}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.437, "count": 172, "max_ms": 160.944, "p95_ms": 154.777}, "read_interval": {"avg_ms": 147.562, "count": 172, "max_ms": 172.85, "p95_ms": 166.702}, "triton_wait": {"avg_ms": 65.617, "count": 172, "max_ms": 89.456, "p95_ms": 84.953}, "visualize": {"avg_ms": 4.227, "count": 172, "max_ms": 4.634, "p95_ms": 4.488}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11430}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10705.145, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d9c70ca980] error while decoding MB 9 18, bytestream -26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 424, "h264_decode_error_count": 1315, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:17:47+09:00", "last_error_summary": "[h264 @ 0x55d9c70ca980] error while decoding MB 9 18, bytestream -26", "last_exit_at": "2026-07-01T07:15:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.476, "last_started_at": "2026-07-01T07:15:03+09:00", "pid": 86433, "process": {"exists": false, "pid": 86433}, "restart_count": 464, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 465}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.938, "sample_count": 2018, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1371693, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:17:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-01T07:17:47+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-01T07:17:47+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T07:17:47.103936+09:00", "mtime_epoch": 1782857867.1039364, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189417}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:17:47+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5019.656, "name": "celery", "pid": 267, "rss_mb": 5019.656, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.757, "pushed": 6.757, "read": 6.757}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.334, "count": 202, "max_ms": 42.156, "p95_ms": 37.877}, "gstreamer_push": {"avg_ms": 1.107, "count": 202, "max_ms": 1.405, "p95_ms": 1.168}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.458, "count": 202, "max_ms": 44.629, "p95_ms": 33.976}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.845, "count": 202, "max_ms": 84.253, "p95_ms": 82.678}, "yolo11_640_fire_11class": {"avg_ms": 51.546, "count": 202, "max_ms": 69.61, "p95_ms": 67.243}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.304, "count": 202, "max_ms": 81.218, "p95_ms": 79.229}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.535, "count": 202, "max_ms": 156.488, "p95_ms": 152.351}, "read_interval": {"avg_ms": 148.359, "count": 202, "max_ms": 169.81, "p95_ms": 165.53}, "triton_wait": {"avg_ms": 65.237, "count": 202, "max_ms": 84.265, "p95_ms": 82.691}, "visualize": {"avg_ms": 4.314, "count": 202, "max_ms": 6.598, "p95_ms": 4.881}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11430}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10704.633, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:17:47+09:00", "last_error_summary": "[h264 @ 0x55d9c70ca980] error while decoding MB 9 18, bytestream -26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 424, "h264_decode_error_count": 1315, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:17:47+09:00", "last_error_summary": "[h264 @ 0x55d9c70ca980] error while decoding MB 9 18, bytestream -26", "last_exit_at": "2026-07-01T07:17:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.574, "last_started_at": "2026-07-01T07:15:03+09:00", "pid": 86433, "process": {"exists": false, "pid": 86433}, "restart_count": 464, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 465}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.938, "sample_count": 2018, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1371694, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:17:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:17:47+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-01T07:17:47+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-01T07:17:47.103936+09:00", "mtime_epoch": 1782857867.1039364, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189417}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:17:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5019.656, "name": "celery", "pid": 267, "rss_mb": 5019.656, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.768, "pushed": 6.768, "read": 6.735}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.34, "count": 202, "max_ms": 42.156, "p95_ms": 37.877}, "gstreamer_push": {"avg_ms": 1.107, "count": 203, "max_ms": 1.405, "p95_ms": 1.168}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.451, "count": 202, "max_ms": 44.629, "p95_ms": 33.976}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.934, "count": 202, "max_ms": 84.253, "p95_ms": 82.678}, "yolo11_640_fire_11class": {"avg_ms": 51.55, "count": 202, "max_ms": 69.61, "p95_ms": 67.243}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.421, "count": 202, "max_ms": 81.218, "p95_ms": 79.229}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.596, "count": 203, "max_ms": 156.488, "p95_ms": 152.351}, "read_interval": {"avg_ms": 148.359, "count": 202, "max_ms": 169.81, "p95_ms": 165.53}, "triton_wait": {"avg_ms": 65.325, "count": 202, "max_ms": 84.265, "p95_ms": 82.691}, "visualize": {"avg_ms": 4.314, "count": 202, "max_ms": 6.598, "p95_ms": 4.881}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11430}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10704.625, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 424, "h264_decode_error_count": 1315, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:17:47+09:00", "last_error_summary": "[h264 @ 0x55d9c70ca980] error while decoding MB 9 18, bytestream -26", "last_exit_at": "2026-07-01T07:17:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.579, "last_started_at": "2026-07-01T07:15:03+09:00", "pid": 86433, "process": {"exists": false, "pid": 86433}, "restart_count": 464, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 465}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.938, "sample_count": 2018, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1371694, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:17:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:17:47+09:00", "last_frame_read_age_sec": 0.16, "last_frame_read_at": "2026-07-01T07:17:47+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-01T07:17:47.103936+09:00", "mtime_epoch": 1782857867.1039364, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189417}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:17:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5019.656, "name": "celery", "pid": 267, "rss_mb": 5019.656, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.767, "pushed": 6.767, "read": 6.733}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.34, "count": 202, "max_ms": 42.156, "p95_ms": 37.877}, "gstreamer_push": {"avg_ms": 1.107, "count": 203, "max_ms": 1.405, "p95_ms": 1.168}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.451, "count": 202, "max_ms": 44.629, "p95_ms": 33.976}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.934, "count": 202, "max_ms": 84.253, "p95_ms": 82.678}, "yolo11_640_fire_11class": {"avg_ms": 51.55, "count": 202, "max_ms": 69.61, "p95_ms": 67.243}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.421, "count": 202, "max_ms": 81.218, "p95_ms": 79.229}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.596, "count": 203, "max_ms": 156.488, "p95_ms": 152.351}, "read_interval": {"avg_ms": 148.359, "count": 202, "max_ms": 169.81, "p95_ms": 165.53}, "triton_wait": {"avg_ms": 65.325, "count": 202, "max_ms": 84.265, "p95_ms": 82.691}, "visualize": {"avg_ms": 4.314, "count": 202, "max_ms": 6.598, "p95_ms": 4.881}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11430}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10809.961, "limit_mb": 8796093022207.996}, "detail": {"pid": 86540, "restart_count": 465, "start_count": 466}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 424, "h264_decode_error_count": 1315, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:17:47+09:00", "last_error_summary": "[h264 @ 0x55d9c70ca980] error while decoding MB 9 18, bytestream -26", "last_exit_at": "2026-07-01T07:17:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:17:50+09:00", "pid": 86540, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.188, "name": "ffmpeg", "pid": 86540, "rss_mb": 136.188, "state": "S (sleeping)", "threads": 106}, "restart_count": 465, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 466}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.938, "sample_count": 1984, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1371694, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:17:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.927, "last_frame_pushed_at": "2026-07-01T07:17:47+09:00", "last_frame_read_age_sec": 5.076, "last_frame_read_at": "2026-07-01T07:17:47+09:00", "mjpeg": {"age_sec": 4.909, "exists": true, "mtime": "2026-07-01T07:17:47.259939+09:00", "mtime_epoch": 1782857867.2599392, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189806}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:17:52+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5019.656, "name": "celery", "pid": 267, "rss_mb": 5019.656, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 169, "pushed_frames": 169, "read_frames": 168, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.654, "pushed": 5.654, "read": 5.62}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.475, "count": 169, "max_ms": 42.156, "p95_ms": 37.941}, "gstreamer_push": {"avg_ms": 1.103, "count": 169, "max_ms": 1.32, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.539, "count": 169, "max_ms": 44.629, "p95_ms": 34.39}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.953, "count": 169, "max_ms": 84.253, "p95_ms": 82.707}, "yolo11_640_fire_11class": {"avg_ms": 51.742, "count": 169, "max_ms": 69.61, "p95_ms": 67.847}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.593, "count": 169, "max_ms": 81.218, "p95_ms": 79.229}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.814, "count": 169, "max_ms": 155.541, "p95_ms": 152.253}, "read_interval": {"avg_ms": 148.624, "count": 168, "max_ms": 169.81, "p95_ms": 165.326}, "triton_wait": {"avg_ms": 65.41, "count": 169, "max_ms": 84.265, "p95_ms": 82.719}, "visualize": {"avg_ms": 4.323, "count": 169, "max_ms": 6.598, "p95_ms": 4.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11440}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10707.172, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e2151d1f80] error while decoding MB 44 5, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 425, "h264_decode_error_count": 1316, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:20:33+09:00", "last_error_summary": "[h264 @ 0x55e2151d1f80] error while decoding MB 44 5, bytestream -5", "last_exit_at": "2026-07-01T07:17:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.386, "last_started_at": "2026-07-01T07:17:50+09:00", "pid": 86540, "process": {"exists": false, "pid": 86540}, "restart_count": 465, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 466}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.601, "sample_count": 2019, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1372801, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:20:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T07:20:33+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T07:20:33+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T07:20:33.510894+09:00", "mtime_epoch": 1782858033.5108943, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186326}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:20:33+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5020.172, "name": "celery", "pid": 267, "rss_mb": 5020.172, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.897, "pushed": 6.897, "read": 6.897}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.229, "count": 206, "max_ms": 39.184, "p95_ms": 36.28}, "gstreamer_push": {"avg_ms": 1.087, "count": 206, "max_ms": 1.36, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.694, "count": 206, "max_ms": 43.717, "p95_ms": 35.081}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.094, "count": 206, "max_ms": 87.73, "p95_ms": 82.514}, "yolo11_640_fire_11class": {"avg_ms": 49.719, "count": 206, "max_ms": 66.275, "p95_ms": 64.165}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.267, "count": 206, "max_ms": 82.565, "p95_ms": 78.596}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.219, "count": 206, "max_ms": 160.155, "p95_ms": 153.611}, "read_interval": {"avg_ms": 145.435, "count": 206, "max_ms": 172.966, "p95_ms": 166.342}, "triton_wait": {"avg_ms": 63.259, "count": 206, "max_ms": 87.741, "p95_ms": 82.526}, "visualize": {"avg_ms": 4.2, "count": 206, "max_ms": 5.345, "p95_ms": 4.596}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11440}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10706.586, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:20:33+09:00", "last_error_summary": "[h264 @ 0x55e2151d1f80] error while decoding MB 44 5, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 425, "h264_decode_error_count": 1316, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:20:33+09:00", "last_error_summary": "[h264 @ 0x55e2151d1f80] error while decoding MB 44 5, bytestream -5", "last_exit_at": "2026-07-01T07:20:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.482, "last_started_at": "2026-07-01T07:17:50+09:00", "pid": 86540, "process": {"exists": false, "pid": 86540}, "restart_count": 465, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 466}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.601, "sample_count": 2019, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1372802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:20:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:20:33+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-01T07:20:33+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-01T07:20:33.510894+09:00", "mtime_epoch": 1782858033.5108943, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186326}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:20:33+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5020.172, "name": "celery", "pid": 267, "rss_mb": 5020.172, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.908, "pushed": 6.908, "read": 6.875}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.22, "count": 207, "max_ms": 39.184, "p95_ms": 36.28}, "gstreamer_push": {"avg_ms": 1.087, "count": 207, "max_ms": 1.36, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.702, "count": 207, "max_ms": 43.717, "p95_ms": 35.081}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.137, "count": 207, "max_ms": 87.73, "p95_ms": 82.514}, "yolo11_640_fire_11class": {"avg_ms": 49.77, "count": 207, "max_ms": 66.275, "p95_ms": 64.165}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.336, "count": 207, "max_ms": 82.565, "p95_ms": 78.596}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.257, "count": 207, "max_ms": 160.155, "p95_ms": 153.611}, "read_interval": {"avg_ms": 145.435, "count": 206, "max_ms": 172.966, "p95_ms": 166.342}, "triton_wait": {"avg_ms": 63.304, "count": 207, "max_ms": 87.741, "p95_ms": 82.526}, "visualize": {"avg_ms": 4.2, "count": 207, "max_ms": 5.345, "p95_ms": 4.596}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11440}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10706.578, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 425, "h264_decode_error_count": 1316, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:20:33+09:00", "last_error_summary": "[h264 @ 0x55e2151d1f80] error while decoding MB 44 5, bytestream -5", "last_exit_at": "2026-07-01T07:20:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.487, "last_started_at": "2026-07-01T07:17:50+09:00", "pid": 86540, "process": {"exists": false, "pid": 86540}, "restart_count": 465, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 466}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.601, "sample_count": 2019, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1372802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:20:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:20:33+09:00", "last_frame_read_age_sec": 0.153, "last_frame_read_at": "2026-07-01T07:20:33+09:00", "mjpeg": {"age_sec": 0.144, "exists": true, "mtime": "2026-07-01T07:20:33.510894+09:00", "mtime_epoch": 1782858033.5108943, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186326}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:20:33+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5020.172, "name": "celery", "pid": 267, "rss_mb": 5020.172, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.907, "pushed": 6.907, "read": 6.874}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.218, "count": 206, "max_ms": 39.184, "p95_ms": 36.28}, "gstreamer_push": {"avg_ms": 1.087, "count": 207, "max_ms": 1.36, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.694, "count": 206, "max_ms": 43.717, "p95_ms": 35.081}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.09, "count": 206, "max_ms": 87.73, "p95_ms": 82.514}, "yolo11_640_fire_11class": {"avg_ms": 49.755, "count": 206, "max_ms": 66.275, "p95_ms": 64.165}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.288, "count": 206, "max_ms": 82.565, "p95_ms": 78.596}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.257, "count": 207, "max_ms": 160.155, "p95_ms": 153.611}, "read_interval": {"avg_ms": 145.435, "count": 206, "max_ms": 172.966, "p95_ms": 166.342}, "triton_wait": {"avg_ms": 63.259, "count": 206, "max_ms": 87.741, "p95_ms": 82.526}, "visualize": {"avg_ms": 4.2, "count": 206, "max_ms": 5.345, "p95_ms": 4.596}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11440}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10814.25, "limit_mb": 8796093022207.996}, "detail": {"pid": 86647, "restart_count": 466, "start_count": 467}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 425, "h264_decode_error_count": 1316, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:20:33+09:00", "last_error_summary": "[h264 @ 0x55e2151d1f80] error while decoding MB 44 5, bytestream -5", "last_exit_at": "2026-07-01T07:20:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:20:36+09:00", "pid": 86647, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.562, "name": "ffmpeg", "pid": 86647, "rss_mb": 138.562, "state": "S (sleeping)", "threads": 106}, "restart_count": 466, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 467}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.601, "sample_count": 1989, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1372802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:20:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.409, "last_frame_pushed_at": "2026-07-01T07:20:33+09:00", "last_frame_read_age_sec": 4.552, "last_frame_read_at": "2026-07-01T07:20:33+09:00", "mjpeg": {"age_sec": 4.392, "exists": true, "mtime": "2026-07-01T07:20:33.662897+09:00", "mtime_epoch": 1782858033.6628969, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176605}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:20:38+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5020.172, "name": "celery", "pid": 267, "rss_mb": 5020.172, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.91, "pushed": 5.91, "read": 5.877}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.228, "count": 177, "max_ms": 39.184, "p95_ms": 36.589}, "gstreamer_push": {"avg_ms": 1.086, "count": 177, "max_ms": 1.36, "p95_ms": 1.16}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.741, "count": 177, "max_ms": 43.717, "p95_ms": 35.081}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.785, "count": 177, "max_ms": 87.73, "p95_ms": 82.514}, "yolo11_640_fire_11class": {"avg_ms": 49.393, "count": 177, "max_ms": 66.275, "p95_ms": 64.327}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.925, "count": 177, "max_ms": 82.565, "p95_ms": 79.193}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.846, "count": 177, "max_ms": 160.155, "p95_ms": 154.212}, "read_interval": {"avg_ms": 144.988, "count": 176, "max_ms": 172.966, "p95_ms": 167.288}, "triton_wait": {"avg_ms": 62.907, "count": 177, "max_ms": 87.741, "p95_ms": 82.526}, "visualize": {"avg_ms": 4.188, "count": 177, "max_ms": 5.345, "p95_ms": 4.596}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11449}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10708.207, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562fee6cd080] cabac decode of qscale diff failed at 13 38 [h264 @ 0x562fee6cd080] error while decoding MB 13 38, bytestream 3067 [h264 @ 0x562feda5fcc0] error while decoding MB 20 9, bytestream -20", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 426, "h264_decode_error_count": 1319, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:23:27+09:00", "last_error_summary": "[h264 @ 0x562fee6cd080] cabac decode of qscale diff failed at 13 38 [h264 @ 0x562fee6cd080] error while decoding MB 13 38, bytestream 3067 [h264 @ 0x562feda5fcc0] error while decoding MB 20 9, bytestream -20", "last_exit_at": "2026-07-01T07:20:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 171.085, "last_started_at": "2026-07-01T07:20:36+09:00", "pid": 86647, "process": {"exists": false, "pid": 86647}, "restart_count": 466, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 467}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.655, "sample_count": 2028, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1373961, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:23:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T07:23:27+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T07:23:27+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T07:23:27.093922+09:00", "mtime_epoch": 1782858207.0939221, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184395}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:23:27+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5020.688, "name": "celery", "pid": 267, "rss_mb": 5020.688, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.791, "pushed": 6.791, "read": 6.791}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.282, "count": 203, "max_ms": 39.417, "p95_ms": 36.818}, "gstreamer_push": {"avg_ms": 1.123, "count": 203, "max_ms": 1.526, "p95_ms": 1.289}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.539, "count": 203, "max_ms": 36.204, "p95_ms": 33.245}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.861, "count": 203, "max_ms": 89.578, "p95_ms": 82.651}, "yolo11_640_fire_11class": {"avg_ms": 50.699, "count": 203, "max_ms": 71.214, "p95_ms": 66.824}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.605, "count": 203, "max_ms": 81.872, "p95_ms": 77.205}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.105, "count": 203, "max_ms": 158.643, "p95_ms": 152.464}, "read_interval": {"avg_ms": 147.639, "count": 203, "max_ms": 173.755, "p95_ms": 166.143}, "triton_wait": {"avg_ms": 65.116, "count": 203, "max_ms": 89.591, "p95_ms": 82.664}, "visualize": {"avg_ms": 4.278, "count": 203, "max_ms": 5.648, "p95_ms": 4.794}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11449}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10708.137, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:23:27+09:00", "last_error_summary": "[h264 @ 0x562fee6cd080] cabac decode of qscale diff failed at 13 38 [h264 @ 0x562fee6cd080] error while decoding MB 13 38, bytestream 3067 [h264 @ 0x562feda5fcc0] error while decoding MB 20 9, bytestream -20", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 426, "h264_decode_error_count": 1319, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:23:27+09:00", "last_error_summary": "[h264 @ 0x562fee6cd080] cabac decode of qscale diff failed at 13 38 [h264 @ 0x562fee6cd080] error while decoding MB 13 38, bytestream 3067 [h264 @ 0x562feda5fcc0] error while decoding MB 20 9, bytestream -20", "last_exit_at": "2026-07-01T07:23:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 171.183, "last_started_at": "2026-07-01T07:20:36+09:00", "pid": 86647, "process": {"exists": false, "pid": 86647}, "restart_count": 466, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 467}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.655, "sample_count": 2029, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1373962, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:23:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:23:27+09:00", "last_frame_read_age_sec": 0.149, "last_frame_read_at": "2026-07-01T07:23:27+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-01T07:23:27.093922+09:00", "mtime_epoch": 1782858207.0939221, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184395}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:23:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5020.688, "name": "celery", "pid": 267, "rss_mb": 5020.688, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.802, "pushed": 6.802, "read": 6.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.306, "count": 203, "max_ms": 39.417, "p95_ms": 36.818}, "gstreamer_push": {"avg_ms": 1.124, "count": 204, "max_ms": 1.526, "p95_ms": 1.289}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.53, "count": 203, "max_ms": 36.204, "p95_ms": 33.245}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.852, "count": 203, "max_ms": 89.578, "p95_ms": 82.651}, "yolo11_640_fire_11class": {"avg_ms": 50.678, "count": 203, "max_ms": 71.214, "p95_ms": 66.824}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.589, "count": 203, "max_ms": 81.872, "p95_ms": 77.205}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.139, "count": 204, "max_ms": 158.643, "p95_ms": 152.464}, "read_interval": {"avg_ms": 147.639, "count": 203, "max_ms": 173.755, "p95_ms": 166.143}, "triton_wait": {"avg_ms": 65.108, "count": 203, "max_ms": 89.591, "p95_ms": 82.664}, "visualize": {"avg_ms": 4.281, "count": 203, "max_ms": 5.648, "p95_ms": 4.833}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11449}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10708.129, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 426, "h264_decode_error_count": 1319, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:23:27+09:00", "last_error_summary": "[h264 @ 0x562fee6cd080] cabac decode of qscale diff failed at 13 38 [h264 @ 0x562fee6cd080] error while decoding MB 13 38, bytestream 3067 [h264 @ 0x562feda5fcc0] error while decoding MB 20 9, bytestream -20", "last_exit_at": "2026-07-01T07:23:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 171.189, "last_started_at": "2026-07-01T07:20:36+09:00", "pid": 86647, "process": {"exists": false, "pid": 86647}, "restart_count": 466, "restart_count_per_hour": 24, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 467}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.655, "sample_count": 2029, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1373962, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:23:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T07:23:27+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-01T07:23:27+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-01T07:23:27.093922+09:00", "mtime_epoch": 1782858207.0939221, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184395}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:23:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5020.688, "name": "celery", "pid": 267, "rss_mb": 5020.688, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.8, "pushed": 6.8, "read": 6.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.306, "count": 203, "max_ms": 39.417, "p95_ms": 36.818}, "gstreamer_push": {"avg_ms": 1.124, "count": 204, "max_ms": 1.526, "p95_ms": 1.289}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.53, "count": 203, "max_ms": 36.204, "p95_ms": 33.245}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.852, "count": 203, "max_ms": 89.578, "p95_ms": 82.651}, "yolo11_640_fire_11class": {"avg_ms": 50.678, "count": 203, "max_ms": 71.214, "p95_ms": 66.824}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.589, "count": 203, "max_ms": 81.872, "p95_ms": 77.205}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.139, "count": 204, "max_ms": 158.643, "p95_ms": 152.464}, "read_interval": {"avg_ms": 147.639, "count": 203, "max_ms": 173.755, "p95_ms": 166.143}, "triton_wait": {"avg_ms": 65.108, "count": 203, "max_ms": 89.591, "p95_ms": 82.664}, "visualize": {"avg_ms": 4.281, "count": 203, "max_ms": 5.648, "p95_ms": 4.833}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11449}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10812.965, "limit_mb": 8796093022207.996}, "detail": {"pid": 86754, "restart_count": 467, "start_count": 468}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 426, "h264_decode_error_count": 1319, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:23:27+09:00", "last_error_summary": "[h264 @ 0x562fee6cd080] cabac decode of qscale diff failed at 13 38 [h264 @ 0x562fee6cd080] error while decoding MB 13 38, bytestream 3067 [h264 @ 0x562feda5fcc0] error while decoding MB 20 9, bytestream -20", "last_exit_at": "2026-07-01T07:23:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:23:29+09:00", "pid": 86754, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.195, "name": "ffmpeg", "pid": 86754, "rss_mb": 136.195, "state": "S (sleeping)", "threads": 106}, "restart_count": 467, "restart_count_per_hour": 25, "rtsp_timeout_count": 390, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 468}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.673, "sample_count": 2001, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1373962, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:23:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.06, "last_frame_pushed_at": "2026-07-01T07:23:27+09:00", "last_frame_read_age_sec": 4.203, "last_frame_read_at": "2026-07-01T07:23:27+09:00", "mjpeg": {"age_sec": 4.045, "exists": true, "mtime": "2026-07-01T07:23:27.245925+09:00", "mtime_epoch": 1782858207.245925, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184485}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:23:31+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5020.688, "name": "celery", "pid": 267, "rss_mb": 5020.688, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.902, "pushed": 5.902, "read": 5.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.272, "count": 176, "max_ms": 39.417, "p95_ms": 36.912}, "gstreamer_push": {"avg_ms": 1.119, "count": 177, "max_ms": 1.514, "p95_ms": 1.207}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.555, "count": 176, "max_ms": 36.204, "p95_ms": 33.38}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.522, "count": 176, "max_ms": 89.578, "p95_ms": 82.797}, "yolo11_640_fire_11class": {"avg_ms": 50.661, "count": 176, "max_ms": 71.214, "p95_ms": 66.874}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.276, "count": 176, "max_ms": 81.872, "p95_ms": 77.186}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.794, "count": 177, "max_ms": 158.643, "p95_ms": 152.906}, "read_interval": {"avg_ms": 147.234, "count": 176, "max_ms": 173.755, "p95_ms": 166.496}, "triton_wait": {"avg_ms": 64.811, "count": 176, "max_ms": 89.591, "p95_ms": 82.81}, "visualize": {"avg_ms": 4.279, "count": 176, "max_ms": 5.168, "p95_ms": 4.833}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11458}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10709.977, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ecca432300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca432300] error while decoding MB 0 4, bytestream 1610 [h264 @ 0x55eccb0b4d40] cabac decode of qscale diff failed at 60 24 [h264 @ 0x55eccb0b4d40] error while decoding MB 60 24, bytestream 6043 [h264 @ 0x55ecca425fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca425fc0] error while decoding MB 0 29, bytestream 96908 [h264 @ 0x55eccacaa580] top block unavailable for requested intra", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 427, "h264_decode_error_count": 1324, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:26:20+09:00", "last_error_summary": "[h264 @ 0x55ecca432300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca432300] error while decoding MB 0 4, bytestream 1610 [h264 @ 0x55eccb0b4d40] cabac decode of qscale diff failed at 60 24 [h264 @ 0x55eccb0b4d40] error while decoding MB 60 24, bytestream 6043 [h264 @ 0x55ecca425fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca425fc0] error while decoding MB 0 29, bytestream 96908 [h264 @ 0x55eccacaa580] top block unavailable for requested intra", "last_exit_at": "2026-07-01T07:23:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 171.3, "last_started_at": "2026-07-01T07:23:29+09:00", "pid": 86754, "process": {"exists": false, "pid": 86754}, "restart_count": 467, "restart_count_per_hour": 24, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 468}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.896, "sample_count": 1901, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1374995, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:26:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T07:26:20+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T07:26:20+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T07:26:20.548906+09:00", "mtime_epoch": 1782858380.548906, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183274}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:26:20+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5021.203, "name": "celery", "pid": 267, "rss_mb": 5021.203, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 76, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.541, "pushed": 2.541, "read": 2.541}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.673, "count": 76, "max_ms": 34.968, "p95_ms": 34.375}, "gstreamer_push": {"avg_ms": 1.093, "count": 76, "max_ms": 1.349, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.08, "count": 76, "max_ms": 34.617, "p95_ms": 33.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.854, "count": 76, "max_ms": 88.387, "p95_ms": 84.212}, "yolo11_640_fire_11class": {"avg_ms": 51.703, "count": 76, "max_ms": 67.028, "p95_ms": 64.378}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.178, "count": 76, "max_ms": 82.094, "p95_ms": 80.483}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.086, "count": 76, "max_ms": 156.977, "p95_ms": 153.844}, "read_interval": {"avg_ms": 394.586, "count": 76, "max_ms": 18817.552, "p95_ms": 168.482}, "triton_wait": {"avg_ms": 67.036, "count": 76, "max_ms": 88.399, "p95_ms": 84.225}, "visualize": {"avg_ms": 4.273, "count": 76, "max_ms": 4.572, "p95_ms": 4.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11458}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10709.32, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:26:20+09:00", "last_error_summary": "[h264 @ 0x55ecca432300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca432300] error while decoding MB 0 4, bytestream 1610 [h264 @ 0x55eccb0b4d40] cabac decode of qscale diff failed at 60 24 [h264 @ 0x55eccb0b4d40] error while decoding MB 60 24, bytestream 6043 [h264 @ 0x55ecca425fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca425fc0] error while decoding MB 0 29, bytestream 96908 [h264 @ 0x55eccacaa580] top block unavailable for requested intra", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 427, "h264_decode_error_count": 1324, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:26:20+09:00", "last_error_summary": "[h264 @ 0x55ecca432300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca432300] error while decoding MB 0 4, bytestream 1610 [h264 @ 0x55eccb0b4d40] cabac decode of qscale diff failed at 60 24 [h264 @ 0x55eccb0b4d40] error while decoding MB 60 24, bytestream 6043 [h264 @ 0x55ecca425fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca425fc0] error while decoding MB 0 29, bytestream 96908 [h264 @ 0x55eccacaa580] top block unavailable for requested intra", "last_exit_at": "2026-07-01T07:26:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 171.389, "last_started_at": "2026-07-01T07:23:29+09:00", "pid": 86754, "process": {"exists": false, "pid": 86754}, "restart_count": 467, "restart_count_per_hour": 24, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 468}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.896, "sample_count": 1901, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1374996, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:26:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T07:26:20+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-01T07:26:20+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T07:26:20.548906+09:00", "mtime_epoch": 1782858380.548906, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183274}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:26:20+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5021.203, "name": "celery", "pid": 267, "rss_mb": 5021.203, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 77, "pushed_frames": 77, "read_frames": 76, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.567, "pushed": 2.567, "read": 2.533}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.68, "count": 76, "max_ms": 34.968, "p95_ms": 34.375}, "gstreamer_push": {"avg_ms": 1.093, "count": 77, "max_ms": 1.349, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.092, "count": 76, "max_ms": 34.617, "p95_ms": 33.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.914, "count": 76, "max_ms": 88.387, "p95_ms": 84.212}, "yolo11_640_fire_11class": {"avg_ms": 51.783, "count": 76, "max_ms": 67.028, "p95_ms": 64.378}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.252, "count": 76, "max_ms": 82.094, "p95_ms": 80.483}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.055, "count": 77, "max_ms": 156.977, "p95_ms": 153.844}, "read_interval": {"avg_ms": 394.586, "count": 76, "max_ms": 18817.552, "p95_ms": 168.482}, "triton_wait": {"avg_ms": 67.097, "count": 76, "max_ms": 88.399, "p95_ms": 84.225}, "visualize": {"avg_ms": 4.276, "count": 76, "max_ms": 4.574, "p95_ms": 4.535}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11458}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10709.438, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 427, "h264_decode_error_count": 1324, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:26:20+09:00", "last_error_summary": "[h264 @ 0x55ecca432300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca432300] error while decoding MB 0 4, bytestream 1610 [h264 @ 0x55eccb0b4d40] cabac decode of qscale diff failed at 60 24 [h264 @ 0x55eccb0b4d40] error while decoding MB 60 24, bytestream 6043 [h264 @ 0x55ecca425fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca425fc0] error while decoding MB 0 29, bytestream 96908 [h264 @ 0x55eccacaa580] top block unavailable for requested intra", "last_exit_at": "2026-07-01T07:26:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 171.394, "last_started_at": "2026-07-01T07:23:29+09:00", "pid": 86754, "process": {"exists": false, "pid": 86754}, "restart_count": 467, "restart_count_per_hour": 24, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 468}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.896, "sample_count": 1901, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1374996, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:26:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T07:26:20+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-01T07:26:20+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T07:26:20.548906+09:00", "mtime_epoch": 1782858380.548906, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183274}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:26:20+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5021.203, "name": "celery", "pid": 267, "rss_mb": 5021.203, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 76, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.534, "pushed": 2.534, "read": 2.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.68, "count": 76, "max_ms": 34.968, "p95_ms": 34.375}, "gstreamer_push": {"avg_ms": 1.094, "count": 76, "max_ms": 1.349, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.092, "count": 76, "max_ms": 34.617, "p95_ms": 33.175}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.914, "count": 76, "max_ms": 88.387, "p95_ms": 84.212}, "yolo11_640_fire_11class": {"avg_ms": 51.783, "count": 76, "max_ms": 67.028, "p95_ms": 64.378}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.252, "count": 76, "max_ms": 82.094, "p95_ms": 80.483}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.148, "count": 76, "max_ms": 156.977, "p95_ms": 153.844}, "read_interval": {"avg_ms": 394.586, "count": 76, "max_ms": 18817.552, "p95_ms": 168.482}, "triton_wait": {"avg_ms": 67.097, "count": 76, "max_ms": 88.399, "p95_ms": 84.225}, "visualize": {"avg_ms": 4.276, "count": 76, "max_ms": 4.574, "p95_ms": 4.535}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11458}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10817.164, "limit_mb": 8796093022207.996}, "detail": {"pid": 86861, "restart_count": 468, "start_count": 469}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 427, "h264_decode_error_count": 1324, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:26:20+09:00", "last_error_summary": "[h264 @ 0x55ecca432300] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca432300] error while decoding MB 0 4, bytestream 1610 [h264 @ 0x55eccb0b4d40] cabac decode of qscale diff failed at 60 24 [h264 @ 0x55eccb0b4d40] error while decoding MB 60 24, bytestream 6043 [h264 @ 0x55ecca425fc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ecca425fc0] error while decoding MB 0 29, bytestream 96908 [h264 @ 0x55eccacaa580] top block unavailable for requested intra", "last_exit_at": "2026-07-01T07:26:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:26:22+09:00", "pid": 86861, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.98, "name": "ffmpeg", "pid": 86861, "rss_mb": 138.98, "state": "S (sleeping)", "threads": 106}, "restart_count": 468, "restart_count_per_hour": 25, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 469}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.906, "sample_count": 1874, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1374996, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:26:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.05, "last_frame_pushed_at": "2026-07-01T07:26:20+09:00", "last_frame_read_age_sec": 4.187, "last_frame_read_at": "2026-07-01T07:26:20+09:00", "mjpeg": {"age_sec": 4.037, "exists": true, "mtime": "2026-07-01T07:26:20.688909+09:00", "mtime_epoch": 1782858380.6889086, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183622}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:26:24+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5021.203, "name": "celery", "pid": 267, "rss_mb": 5021.203, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.64, "pushed": 1.64, "read": 1.607}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.691, "count": 49, "max_ms": 34.968, "p95_ms": 34.537}, "gstreamer_push": {"avg_ms": 1.096, "count": 49, "max_ms": 1.349, "p95_ms": 1.139}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.001, "count": 49, "max_ms": 33.336, "p95_ms": 32.929}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.147, "count": 49, "max_ms": 88.387, "p95_ms": 84.212}, "yolo11_640_fire_11class": {"avg_ms": 51.808, "count": 49, "max_ms": 67.028, "p95_ms": 64.378}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.162, "count": 49, "max_ms": 81.795, "p95_ms": 79.762}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.178, "count": 49, "max_ms": 156.977, "p95_ms": 153.391}, "read_interval": {"avg_ms": 537.831, "count": 48, "max_ms": 18817.552, "p95_ms": 168.482}, "triton_wait": {"avg_ms": 67.159, "count": 49, "max_ms": 88.399, "p95_ms": 84.225}, "visualize": {"avg_ms": 4.273, "count": 49, "max_ms": 4.574, "p95_ms": 4.476}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10711.273, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b65d45cbc0] error while decoding MB 22 39, bytestream -36", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 428, "h264_decode_error_count": 1325, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:29:05+09:00", "last_error_summary": "[h264 @ 0x55b65d45cbc0] error while decoding MB 22 39, bytestream -36", "last_exit_at": "2026-07-01T07:26:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.154, "last_started_at": "2026-07-01T07:26:22+09:00", "pid": 86861, "process": {"exists": false, "pid": 86861}, "restart_count": 468, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 469}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.222, "sample_count": 1888, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1376090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:29:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T07:29:05+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T07:29:05+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T07:29:05.835721+09:00", "mtime_epoch": 1782858545.8357215, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:29:05+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5021.719, "name": "celery", "pid": 267, "rss_mb": 5021.719, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.729, "pushed": 6.729, "read": 6.729}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.665, "count": 201, "max_ms": 38.339, "p95_ms": 35.406}, "gstreamer_push": {"avg_ms": 1.104, "count": 201, "max_ms": 1.662, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.131, "count": 201, "max_ms": 39.852, "p95_ms": 33.334}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.889, "count": 201, "max_ms": 84.97, "p95_ms": 82.314}, "yolo11_640_fire_11class": {"avg_ms": 52.727, "count": 201, "max_ms": 73.502, "p95_ms": 67.259}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.874, "count": 201, "max_ms": 79.378, "p95_ms": 75.767}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.719, "count": 201, "max_ms": 156.667, "p95_ms": 153.11}, "read_interval": {"avg_ms": 149.092, "count": 201, "max_ms": 169.297, "p95_ms": 165.91}, "triton_wait": {"avg_ms": 65.915, "count": 201, "max_ms": 84.981, "p95_ms": 82.326}, "visualize": {"avg_ms": 4.287, "count": 201, "max_ms": 5.111, "p95_ms": 4.559}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10710.793, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:29:05+09:00", "last_error_summary": "[h264 @ 0x55b65d45cbc0] error while decoding MB 22 39, bytestream -36", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 428, "h264_decode_error_count": 1325, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:29:05+09:00", "last_error_summary": "[h264 @ 0x55b65d45cbc0] error while decoding MB 22 39, bytestream -36", "last_exit_at": "2026-07-01T07:29:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.261, "last_started_at": "2026-07-01T07:26:22+09:00", "pid": 86861, "process": {"exists": false, "pid": 86861}, "restart_count": 468, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 469}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.222, "sample_count": 1889, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1376091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:29:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:29:05+09:00", "last_frame_read_age_sec": 0.158, "last_frame_read_at": "2026-07-01T07:29:05+09:00", "mjpeg": {"age_sec": 0.151, "exists": true, "mtime": "2026-07-01T07:29:05.835721+09:00", "mtime_epoch": 1782858545.8357215, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:29:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5021.719, "name": "celery", "pid": 267, "rss_mb": 5021.719, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.739, "pushed": 6.739, "read": 6.706}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.668, "count": 201, "max_ms": 38.339, "p95_ms": 35.406}, "gstreamer_push": {"avg_ms": 1.104, "count": 202, "max_ms": 1.662, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.13, "count": 201, "max_ms": 39.852, "p95_ms": 33.334}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.899, "count": 201, "max_ms": 84.97, "p95_ms": 82.526}, "yolo11_640_fire_11class": {"avg_ms": 52.744, "count": 201, "max_ms": 73.502, "p95_ms": 67.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.91, "count": 201, "max_ms": 79.378, "p95_ms": 75.772}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.795, "count": 202, "max_ms": 156.667, "p95_ms": 153.11}, "read_interval": {"avg_ms": 149.092, "count": 201, "max_ms": 169.297, "p95_ms": 165.91}, "triton_wait": {"avg_ms": 65.925, "count": 201, "max_ms": 84.981, "p95_ms": 82.538}, "visualize": {"avg_ms": 4.286, "count": 201, "max_ms": 5.111, "p95_ms": 4.559}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10710.785, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 428, "h264_decode_error_count": 1325, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:29:05+09:00", "last_error_summary": "[h264 @ 0x55b65d45cbc0] error while decoding MB 22 39, bytestream -36", "last_exit_at": "2026-07-01T07:29:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.266, "last_started_at": "2026-07-01T07:26:22+09:00", "pid": 86861, "process": {"exists": false, "pid": 86861}, "restart_count": 468, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 469}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.222, "sample_count": 1889, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1376091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:29:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:29:05+09:00", "last_frame_read_age_sec": 0.164, "last_frame_read_at": "2026-07-01T07:29:05+09:00", "mjpeg": {"age_sec": 0.156, "exists": true, "mtime": "2026-07-01T07:29:05.835721+09:00", "mtime_epoch": 1782858545.8357215, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:29:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5021.719, "name": "celery", "pid": 267, "rss_mb": 5021.719, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.738, "pushed": 6.738, "read": 6.704}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.668, "count": 201, "max_ms": 38.339, "p95_ms": 35.406}, "gstreamer_push": {"avg_ms": 1.104, "count": 202, "max_ms": 1.662, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.13, "count": 201, "max_ms": 39.852, "p95_ms": 33.334}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.899, "count": 201, "max_ms": 84.97, "p95_ms": 82.526}, "yolo11_640_fire_11class": {"avg_ms": 52.744, "count": 201, "max_ms": 73.502, "p95_ms": 67.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.91, "count": 201, "max_ms": 79.378, "p95_ms": 75.772}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.795, "count": 202, "max_ms": 156.667, "p95_ms": 153.11}, "read_interval": {"avg_ms": 149.092, "count": 201, "max_ms": 169.297, "p95_ms": 165.91}, "triton_wait": {"avg_ms": 65.925, "count": 201, "max_ms": 84.981, "p95_ms": 82.538}, "visualize": {"avg_ms": 4.286, "count": 201, "max_ms": 5.111, "p95_ms": 4.559}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10818.441, "limit_mb": 8796093022207.996}, "detail": {"pid": 86968, "restart_count": 469, "start_count": 470}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 428, "h264_decode_error_count": 1325, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:29:05+09:00", "last_error_summary": "[h264 @ 0x55b65d45cbc0] error while decoding MB 22 39, bytestream -36", "last_exit_at": "2026-07-01T07:29:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:29:08+09:00", "pid": 86968, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.277, "name": "ffmpeg", "pid": 86968, "rss_mb": 139.277, "state": "S (sleeping)", "threads": 106}, "restart_count": 469, "restart_count_per_hour": 24, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 470}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.222, "sample_count": 1857, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1376091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:29:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.52, "last_frame_pushed_at": "2026-07-01T07:29:05+09:00", "last_frame_read_age_sec": 4.674, "last_frame_read_at": "2026-07-01T07:29:05+09:00", "mjpeg": {"age_sec": 4.502, "exists": true, "mtime": "2026-07-01T07:29:05.999724+09:00", "mtime_epoch": 1782858545.9997244, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183773}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:29:10+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5021.719, "name": "celery", "pid": 267, "rss_mb": 5021.719, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 171, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.739, "pushed": 5.739, "read": 5.706}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.713, "count": 171, "max_ms": 38.339, "p95_ms": 35.936}, "gstreamer_push": {"avg_ms": 1.105, "count": 172, "max_ms": 1.662, "p95_ms": 1.157}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.976, "count": 171, "max_ms": 35.573, "p95_ms": 32.852}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.686, "count": 171, "max_ms": 84.914, "p95_ms": 82.314}, "yolo11_640_fire_11class": {"avg_ms": 52.341, "count": 171, "max_ms": 72.627, "p95_ms": 67.259}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.755, "count": 171, "max_ms": 79.378, "p95_ms": 75.978}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.407, "count": 172, "max_ms": 156.667, "p95_ms": 153.055}, "read_interval": {"avg_ms": 148.677, "count": 171, "max_ms": 169.297, "p95_ms": 165.428}, "triton_wait": {"avg_ms": 65.707, "count": 171, "max_ms": 84.926, "p95_ms": 82.326}, "visualize": {"avg_ms": 4.285, "count": 171, "max_ms": 5.111, "p95_ms": 4.596}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11476}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10712.551, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5623652502c0] error while decoding MB 19 36, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 429, "h264_decode_error_count": 1326, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:31:52+09:00", "last_error_summary": "[h264 @ 0x5623652502c0] error while decoding MB 19 36, bytestream -5", "last_exit_at": "2026-07-01T07:29:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.822, "last_started_at": "2026-07-01T07:29:08+09:00", "pid": 86968, "process": {"exists": false, "pid": 86968}, "restart_count": 469, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 470}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.266, "sample_count": 2007, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1377192, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:31:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T07:31:52+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T07:31:52+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T07:31:52.274907+09:00", "mtime_epoch": 1782858712.2749074, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182290}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:31:52+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5022.234, "name": "celery", "pid": 267, "rss_mb": 5022.234, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.886, "pushed": 6.886, "read": 6.886}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.122, "count": 206, "max_ms": 37.336, "p95_ms": 35.396}, "gstreamer_push": {"avg_ms": 1.1, "count": 206, "max_ms": 1.581, "p95_ms": 1.154}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.386, "count": 206, "max_ms": 43.368, "p95_ms": 34.061}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.203, "count": 206, "max_ms": 85.103, "p95_ms": 82.768}, "yolo11_640_fire_11class": {"avg_ms": 49.663, "count": 206, "max_ms": 65.952, "p95_ms": 62.782}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.054, "count": 206, "max_ms": 80.647, "p95_ms": 76.5}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.06, "count": 206, "max_ms": 155.201, "p95_ms": 151.63}, "read_interval": {"avg_ms": 145.61, "count": 206, "max_ms": 168.52, "p95_ms": 164.218}, "triton_wait": {"avg_ms": 64.434, "count": 206, "max_ms": 85.115, "p95_ms": 82.784}, "visualize": {"avg_ms": 4.252, "count": 206, "max_ms": 6.629, "p95_ms": 4.517}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11476}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10712.188, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:31:52+09:00", "last_error_summary": "[h264 @ 0x5623652502c0] error while decoding MB 19 36, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 429, "h264_decode_error_count": 1326, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:31:52+09:00", "last_error_summary": "[h264 @ 0x5623652502c0] error while decoding MB 19 36, bytestream -5", "last_exit_at": "2026-07-01T07:31:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.907, "last_started_at": "2026-07-01T07:29:08+09:00", "pid": 86968, "process": {"exists": false, "pid": 86968}, "restart_count": 469, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 470}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.266, "sample_count": 2008, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1377193, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:31:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:31:52+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-01T07:31:52+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-01T07:31:52.274907+09:00", "mtime_epoch": 1782858712.2749074, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182290}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:31:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5022.234, "name": "celery", "pid": 267, "rss_mb": 5022.234, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.9, "pushed": 6.9, "read": 6.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.112, "count": 206, "max_ms": 37.336, "p95_ms": 35.396}, "gstreamer_push": {"avg_ms": 1.1, "count": 207, "max_ms": 1.581, "p95_ms": 1.154}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.383, "count": 206, "max_ms": 43.368, "p95_ms": 34.061}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.2, "count": 206, "max_ms": 85.103, "p95_ms": 82.768}, "yolo11_640_fire_11class": {"avg_ms": 49.659, "count": 206, "max_ms": 65.952, "p95_ms": 62.782}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.072, "count": 206, "max_ms": 80.647, "p95_ms": 76.5}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.042, "count": 207, "max_ms": 155.201, "p95_ms": 151.63}, "read_interval": {"avg_ms": 145.61, "count": 206, "max_ms": 168.52, "p95_ms": 164.218}, "triton_wait": {"avg_ms": 64.431, "count": 206, "max_ms": 85.115, "p95_ms": 82.784}, "visualize": {"avg_ms": 4.248, "count": 206, "max_ms": 6.629, "p95_ms": 4.499}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11476}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10712.129, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 429, "h264_decode_error_count": 1326, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:31:52+09:00", "last_error_summary": "[h264 @ 0x5623652502c0] error while decoding MB 19 36, bytestream -5", "last_exit_at": "2026-07-01T07:31:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.912, "last_started_at": "2026-07-01T07:29:08+09:00", "pid": 86968, "process": {"exists": false, "pid": 86968}, "restart_count": 469, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 470}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.266, "sample_count": 2008, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1377193, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:31:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:31:52+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T07:31:52+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-01T07:31:52.274907+09:00", "mtime_epoch": 1782858712.2749074, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182290}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:31:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5022.234, "name": "celery", "pid": 267, "rss_mb": 5022.234, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.868, "pushed": 6.868, "read": 6.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.112, "count": 206, "max_ms": 37.336, "p95_ms": 35.396}, "gstreamer_push": {"avg_ms": 1.101, "count": 206, "max_ms": 1.581, "p95_ms": 1.154}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.383, "count": 206, "max_ms": 43.368, "p95_ms": 34.061}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.2, "count": 206, "max_ms": 85.103, "p95_ms": 82.768}, "yolo11_640_fire_11class": {"avg_ms": 49.659, "count": 206, "max_ms": 65.952, "p95_ms": 62.782}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.072, "count": 206, "max_ms": 80.647, "p95_ms": 76.5}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.045, "count": 206, "max_ms": 155.201, "p95_ms": 151.63}, "read_interval": {"avg_ms": 145.61, "count": 206, "max_ms": 168.52, "p95_ms": 164.218}, "triton_wait": {"avg_ms": 64.431, "count": 206, "max_ms": 85.115, "p95_ms": 82.784}, "visualize": {"avg_ms": 4.248, "count": 206, "max_ms": 6.629, "p95_ms": 4.499}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11476}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10817.281, "limit_mb": 8796093022207.996}, "detail": {"pid": 87075, "restart_count": 470, "start_count": 471}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 429, "h264_decode_error_count": 1326, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:31:52+09:00", "last_error_summary": "[h264 @ 0x5623652502c0] error while decoding MB 19 36, bytestream -5", "last_exit_at": "2026-07-01T07:31:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:31:55+09:00", "pid": 87075, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.805, "name": "ffmpeg", "pid": 87075, "rss_mb": 136.805, "state": "S (sleeping)", "threads": 106}, "restart_count": 470, "restart_count_per_hour": 24, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 471}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.254, "sample_count": 1978, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1377193, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:31:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.607, "last_frame_pushed_at": "2026-07-01T07:31:52+09:00", "last_frame_read_age_sec": 4.739, "last_frame_read_at": "2026-07-01T07:31:52+09:00", "mjpeg": {"age_sec": 4.591, "exists": true, "mtime": "2026-07-01T07:31:52.418911+09:00", "mtime_epoch": 1782858712.4189112, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182301}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:31:57+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5022.234, "name": "celery", "pid": 267, "rss_mb": 5022.234, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.833, "pushed": 5.833, "read": 5.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.988, "count": 174, "max_ms": 36.169, "p95_ms": 35.195}, "gstreamer_push": {"avg_ms": 1.099, "count": 175, "max_ms": 1.581, "p95_ms": 1.154}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.478, "count": 174, "max_ms": 43.368, "p95_ms": 34.076}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.525, "count": 174, "max_ms": 85.087, "p95_ms": 82.905}, "yolo11_640_fire_11class": {"avg_ms": 49.779, "count": 174, "max_ms": 65.952, "p95_ms": 63.079}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.431, "count": 174, "max_ms": 80.647, "p95_ms": 76.627}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.212, "count": 175, "max_ms": 155.201, "p95_ms": 151.765}, "read_interval": {"avg_ms": 145.882, "count": 174, "max_ms": 168.52, "p95_ms": 164.218}, "triton_wait": {"avg_ms": 64.765, "count": 174, "max_ms": 85.099, "p95_ms": 82.917}, "visualize": {"avg_ms": 4.246, "count": 174, "max_ms": 6.629, "p95_ms": 4.499}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11485}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10714.684, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5628cb906680] cabac decode of qscale diff failed at 37 37 [h264 @ 0x5628cb906680] error while decoding MB 37 37, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 430, "h264_decode_error_count": 1328, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:34:38+09:00", "last_error_summary": "[h264 @ 0x5628cb906680] cabac decode of qscale diff failed at 37 37 [h264 @ 0x5628cb906680] error while decoding MB 37 37, bytestream -5", "last_exit_at": "2026-07-01T07:31:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.611, "last_started_at": "2026-07-01T07:31:55+09:00", "pid": 87075, "process": {"exists": false, "pid": 87075}, "restart_count": 470, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 471}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.949, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1378289, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:34:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T07:34:38+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T07:34:38+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T07:34:38.575272+09:00", "mtime_epoch": 1782858878.575272, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180199}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:34:38+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5022.75, "name": "celery", "pid": 267, "rss_mb": 5022.75, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.821, "pushed": 6.821, "read": 6.821}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.455, "count": 204, "max_ms": 50.039, "p95_ms": 35.297}, "gstreamer_push": {"avg_ms": 1.1, "count": 204, "max_ms": 1.864, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.69, "count": 204, "max_ms": 41.492, "p95_ms": 34.207}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.195, "count": 204, "max_ms": 90.061, "p95_ms": 82.802}, "yolo11_640_fire_11class": {"avg_ms": 49.766, "count": 204, "max_ms": 70.1, "p95_ms": 63.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.464, "count": 204, "max_ms": 81.31, "p95_ms": 77.329}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.791, "count": 204, "max_ms": 164.905, "p95_ms": 152.422}, "read_interval": {"avg_ms": 146.928, "count": 204, "max_ms": 176.372, "p95_ms": 165.279}, "triton_wait": {"avg_ms": 64.538, "count": 204, "max_ms": 90.074, "p95_ms": 82.815}, "visualize": {"avg_ms": 4.216, "count": 204, "max_ms": 4.731, "p95_ms": 4.514}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11485}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10713.695, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:34:38+09:00", "last_error_summary": "[h264 @ 0x5628cb906680] cabac decode of qscale diff failed at 37 37 [h264 @ 0x5628cb906680] error while decoding MB 37 37, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 430, "h264_decode_error_count": 1328, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:34:38+09:00", "last_error_summary": "[h264 @ 0x5628cb906680] cabac decode of qscale diff failed at 37 37 [h264 @ 0x5628cb906680] error while decoding MB 37 37, bytestream -5", "last_exit_at": "2026-07-01T07:34:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.716, "last_started_at": "2026-07-01T07:31:55+09:00", "pid": 87075, "process": {"exists": false, "pid": 87075}, "restart_count": 470, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 471}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.949, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1378290, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:34:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:34:38+09:00", "last_frame_read_age_sec": 0.157, "last_frame_read_at": "2026-07-01T07:34:38+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-01T07:34:38.575272+09:00", "mtime_epoch": 1782858878.575272, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180199}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:34:38+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5022.75, "name": "celery", "pid": 267, "rss_mb": 5022.75, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.827, "pushed": 6.827, "read": 6.793}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.492, "count": 204, "max_ms": 50.039, "p95_ms": 35.297}, "gstreamer_push": {"avg_ms": 1.101, "count": 204, "max_ms": 1.864, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.693, "count": 204, "max_ms": 41.492, "p95_ms": 34.207}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.313, "count": 204, "max_ms": 90.061, "p95_ms": 82.802}, "yolo11_640_fire_11class": {"avg_ms": 49.866, "count": 204, "max_ms": 70.1, "p95_ms": 63.189}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.608, "count": 204, "max_ms": 81.31, "p95_ms": 77.329}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.956, "count": 204, "max_ms": 164.905, "p95_ms": 152.422}, "read_interval": {"avg_ms": 147.025, "count": 203, "max_ms": 176.372, "p95_ms": 165.279}, "triton_wait": {"avg_ms": 64.656, "count": 204, "max_ms": 90.074, "p95_ms": 82.815}, "visualize": {"avg_ms": 4.219, "count": 204, "max_ms": 4.731, "p95_ms": 4.514}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11485}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10713.688, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 430, "h264_decode_error_count": 1328, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:34:38+09:00", "last_error_summary": "[h264 @ 0x5628cb906680] cabac decode of qscale diff failed at 37 37 [h264 @ 0x5628cb906680] error while decoding MB 37 37, bytestream -5", "last_exit_at": "2026-07-01T07:34:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.721, "last_started_at": "2026-07-01T07:31:55+09:00", "pid": 87075, "process": {"exists": false, "pid": 87075}, "restart_count": 470, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 471}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.949, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1378290, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:34:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:34:38+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-01T07:34:38+09:00", "mjpeg": {"age_sec": 0.154, "exists": true, "mtime": "2026-07-01T07:34:38.575272+09:00", "mtime_epoch": 1782858878.575272, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180199}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:34:38+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5022.75, "name": "celery", "pid": 267, "rss_mb": 5022.75, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.825, "pushed": 6.825, "read": 6.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.492, "count": 204, "max_ms": 50.039, "p95_ms": 35.297}, "gstreamer_push": {"avg_ms": 1.101, "count": 204, "max_ms": 1.864, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.693, "count": 204, "max_ms": 41.492, "p95_ms": 34.207}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.313, "count": 204, "max_ms": 90.061, "p95_ms": 82.802}, "yolo11_640_fire_11class": {"avg_ms": 49.866, "count": 204, "max_ms": 70.1, "p95_ms": 63.189}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.608, "count": 204, "max_ms": 81.31, "p95_ms": 77.329}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.956, "count": 204, "max_ms": 164.905, "p95_ms": 152.422}, "read_interval": {"avg_ms": 147.025, "count": 203, "max_ms": 176.372, "p95_ms": 165.279}, "triton_wait": {"avg_ms": 64.656, "count": 204, "max_ms": 90.074, "p95_ms": 82.815}, "visualize": {"avg_ms": 4.219, "count": 204, "max_ms": 4.731, "p95_ms": 4.514}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11485}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10821.895, "limit_mb": 8796093022207.996}, "detail": {"pid": 87182, "restart_count": 471, "start_count": 472}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 430, "h264_decode_error_count": 1328, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:34:38+09:00", "last_error_summary": "[h264 @ 0x5628cb906680] cabac decode of qscale diff failed at 37 37 [h264 @ 0x5628cb906680] error while decoding MB 37 37, bytestream -5", "last_exit_at": "2026-07-01T07:34:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:34:40+09:00", "pid": 87182, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.875, "name": "ffmpeg", "pid": 87182, "rss_mb": 138.875, "state": "S (sleeping)", "threads": 106}, "restart_count": 471, "restart_count_per_hour": 24, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 472}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.929, "sample_count": 1981, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1378290, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:34:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.234, "last_frame_pushed_at": "2026-07-01T07:34:38+09:00", "last_frame_read_age_sec": 4.386, "last_frame_read_at": "2026-07-01T07:34:38+09:00", "mjpeg": {"age_sec": 4.214, "exists": true, "mtime": "2026-07-01T07:34:38.739276+09:00", "mtime_epoch": 1782858878.7392762, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180393}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:34:42+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5022.75, "name": "celery", "pid": 267, "rss_mb": 5022.75, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 176, "pushed_frames": 176, "read_frames": 175, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.878, "pushed": 5.878, "read": 5.845}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.542, "count": 176, "max_ms": 50.039, "p95_ms": 35.57}, "gstreamer_push": {"avg_ms": 1.1, "count": 176, "max_ms": 1.864, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.67, "count": 176, "max_ms": 41.492, "p95_ms": 34.615}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.916, "count": 176, "max_ms": 90.061, "p95_ms": 82.217}, "yolo11_640_fire_11class": {"avg_ms": 49.59, "count": 176, "max_ms": 70.1, "p95_ms": 63.189}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.345, "count": 176, "max_ms": 81.31, "p95_ms": 77.158}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.667, "count": 176, "max_ms": 164.905, "p95_ms": 152.291}, "read_interval": {"avg_ms": 146.745, "count": 175, "max_ms": 176.372, "p95_ms": 165.279}, "triton_wait": {"avg_ms": 64.312, "count": 176, "max_ms": 90.074, "p95_ms": 82.229}, "visualize": {"avg_ms": 4.212, "count": 176, "max_ms": 4.731, "p95_ms": 4.521}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11496}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10716.5, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c6000afcc0] error while decoding MB 29 7, bytestream -6 [h264 @ 0x55c600068cc0] left block unavailable for requested intra mode [h264 @ 0x55c600068cc0] error while decoding MB 0 41, bytestream 4907 [h264 @ 0x55c600054d40] error while decoding MB 40 20, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 431, "h264_decode_error_count": 1331, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:37:48+09:00", "last_error_summary": "[h264 @ 0x55c6000afcc0] error while decoding MB 29 7, bytestream -6 [h264 @ 0x55c600068cc0] left block unavailable for requested intra mode [h264 @ 0x55c600068cc0] error while decoding MB 0 41, bytestream 4907 [h264 @ 0x55c600054d40] error while decoding MB 40 20, bytestream -5", "last_exit_at": "2026-07-01T07:34:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.654, "last_started_at": "2026-07-01T07:34:40+09:00", "pid": 87182, "process": {"exists": false, "pid": 87182}, "restart_count": 471, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 472}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.216, "sample_count": 2011, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1379555, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:37:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T07:37:48+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T07:37:48+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T07:37:48.563794+09:00", "mtime_epoch": 1782859068.5637941, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:37:48+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5023.523, "name": "celery", "pid": 267, "rss_mb": 5023.523, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.901, "pushed": 6.901, "read": 6.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.077, "count": 206, "max_ms": 36.685, "p95_ms": 34.828}, "gstreamer_push": {"avg_ms": 1.082, "count": 206, "max_ms": 1.293, "p95_ms": 1.179}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.649, "count": 206, "max_ms": 42.572, "p95_ms": 34.463}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.682, "count": 206, "max_ms": 86.12, "p95_ms": 82.899}, "yolo11_640_fire_11class": {"avg_ms": 49.738, "count": 206, "max_ms": 67.35, "p95_ms": 64.392}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.209, "count": 206, "max_ms": 82.637, "p95_ms": 79.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.535, "count": 206, "max_ms": 156.95, "p95_ms": 152.349}, "read_interval": {"avg_ms": 145.391, "count": 206, "max_ms": 168.721, "p95_ms": 164.768}, "triton_wait": {"avg_ms": 63.918, "count": 206, "max_ms": 86.135, "p95_ms": 82.912}, "visualize": {"avg_ms": 4.221, "count": 206, "max_ms": 5.272, "p95_ms": 4.467}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.85, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11496}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10715.75, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:37:48+09:00", "last_error_summary": "[h264 @ 0x55c6000afcc0] error while decoding MB 29 7, bytestream -6 [h264 @ 0x55c600068cc0] left block unavailable for requested intra mode [h264 @ 0x55c600068cc0] error while decoding MB 0 41, bytestream 4907 [h264 @ 0x55c600054d40] error while decoding MB 40 20, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 431, "h264_decode_error_count": 1331, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:37:48+09:00", "last_error_summary": "[h264 @ 0x55c6000afcc0] error while decoding MB 29 7, bytestream -6 [h264 @ 0x55c600068cc0] left block unavailable for requested intra mode [h264 @ 0x55c600068cc0] error while decoding MB 0 41, bytestream 4907 [h264 @ 0x55c600054d40] error while decoding MB 40 20, bytestream -5", "last_exit_at": "2026-07-01T07:37:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.741, "last_started_at": "2026-07-01T07:34:40+09:00", "pid": 87182, "process": {"exists": false, "pid": 87182}, "restart_count": 471, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 472}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.211, "sample_count": 2012, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1379556, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:37:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:37:48+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-01T07:37:48+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T07:37:48.563794+09:00", "mtime_epoch": 1782859068.5637941, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:37:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5023.523, "name": "celery", "pid": 267, "rss_mb": 5023.523, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.914, "pushed": 6.914, "read": 6.881}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.074, "count": 207, "max_ms": 36.685, "p95_ms": 34.828}, "gstreamer_push": {"avg_ms": 1.082, "count": 207, "max_ms": 1.293, "p95_ms": 1.179}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.644, "count": 207, "max_ms": 42.572, "p95_ms": 34.463}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.68, "count": 207, "max_ms": 86.12, "p95_ms": 82.899}, "yolo11_640_fire_11class": {"avg_ms": 49.733, "count": 207, "max_ms": 67.35, "p95_ms": 64.392}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.196, "count": 207, "max_ms": 82.637, "p95_ms": 79.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.531, "count": 207, "max_ms": 156.95, "p95_ms": 152.349}, "read_interval": {"avg_ms": 145.391, "count": 206, "max_ms": 168.721, "p95_ms": 164.768}, "triton_wait": {"avg_ms": 63.915, "count": 207, "max_ms": 86.135, "p95_ms": 82.912}, "visualize": {"avg_ms": 4.222, "count": 207, "max_ms": 5.272, "p95_ms": 4.467}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11496}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10715.797, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 431, "h264_decode_error_count": 1331, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:37:48+09:00", "last_error_summary": "[h264 @ 0x55c6000afcc0] error while decoding MB 29 7, bytestream -6 [h264 @ 0x55c600068cc0] left block unavailable for requested intra mode [h264 @ 0x55c600068cc0] error while decoding MB 0 41, bytestream 4907 [h264 @ 0x55c600054d40] error while decoding MB 40 20, bytestream -5", "last_exit_at": "2026-07-01T07:37:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.747, "last_started_at": "2026-07-01T07:34:40+09:00", "pid": 87182, "process": {"exists": false, "pid": 87182}, "restart_count": 471, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 472}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.211, "sample_count": 2012, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1379556, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:37:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:37:48+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-01T07:37:48+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T07:37:48.563794+09:00", "mtime_epoch": 1782859068.5637941, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:37:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5023.523, "name": "celery", "pid": 267, "rss_mb": 5023.523, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.913, "pushed": 6.913, "read": 6.88}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.074, "count": 207, "max_ms": 36.685, "p95_ms": 34.828}, "gstreamer_push": {"avg_ms": 1.082, "count": 207, "max_ms": 1.293, "p95_ms": 1.179}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.644, "count": 207, "max_ms": 42.572, "p95_ms": 34.463}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.68, "count": 207, "max_ms": 86.12, "p95_ms": 82.899}, "yolo11_640_fire_11class": {"avg_ms": 49.733, "count": 207, "max_ms": 67.35, "p95_ms": 64.392}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.196, "count": 207, "max_ms": 82.637, "p95_ms": 79.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.531, "count": 207, "max_ms": 156.95, "p95_ms": 152.349}, "read_interval": {"avg_ms": 145.391, "count": 206, "max_ms": 168.721, "p95_ms": 164.768}, "triton_wait": {"avg_ms": 63.915, "count": 207, "max_ms": 86.135, "p95_ms": 82.912}, "visualize": {"avg_ms": 4.222, "count": 207, "max_ms": 5.272, "p95_ms": 4.467}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11496}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10823.766, "limit_mb": 8796093022207.996}, "detail": {"pid": 87289, "restart_count": 472, "start_count": 473}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 431, "h264_decode_error_count": 1331, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:37:48+09:00", "last_error_summary": "[h264 @ 0x55c6000afcc0] error while decoding MB 29 7, bytestream -6 [h264 @ 0x55c600068cc0] left block unavailable for requested intra mode [h264 @ 0x55c600068cc0] error while decoding MB 0 41, bytestream 4907 [h264 @ 0x55c600054d40] error while decoding MB 40 20, bytestream -5", "last_exit_at": "2026-07-01T07:37:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:37:51+09:00", "pid": 87289, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.254, "name": "ffmpeg", "pid": 87289, "rss_mb": 139.254, "state": "S (sleeping)", "threads": 106}, "restart_count": 472, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 473}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.211, "sample_count": 1978, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1379556, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:37:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.863, "last_frame_pushed_at": "2026-07-01T07:37:48+09:00", "last_frame_read_age_sec": 4.997, "last_frame_read_at": "2026-07-01T07:37:48+09:00", "mjpeg": {"age_sec": 4.844, "exists": true, "mtime": "2026-07-01T07:37:48.707797+09:00", "mtime_epoch": 1782859068.7077973, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183639}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:37:53+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5023.523, "name": "celery", "pid": 267, "rss_mb": 5023.523, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.82, "pushed": 5.82, "read": 5.787}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.087, "count": 174, "max_ms": 36.044, "p95_ms": 34.91}, "gstreamer_push": {"avg_ms": 1.08, "count": 174, "max_ms": 1.293, "p95_ms": 1.181}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.807, "count": 174, "max_ms": 42.572, "p95_ms": 34.606}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.122, "count": 174, "max_ms": 86.12, "p95_ms": 82.899}, "yolo11_640_fire_11class": {"avg_ms": 49.186, "count": 174, "max_ms": 67.305, "p95_ms": 63.047}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.434, "count": 174, "max_ms": 82.637, "p95_ms": 79.859}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.86, "count": 174, "max_ms": 156.95, "p95_ms": 151.805}, "read_interval": {"avg_ms": 144.652, "count": 173, "max_ms": 168.721, "p95_ms": 164.015}, "triton_wait": {"avg_ms": 63.334, "count": 174, "max_ms": 86.135, "p95_ms": 82.912}, "visualize": {"avg_ms": 4.216, "count": 174, "max_ms": 5.272, "p95_ms": 4.467}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11505}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10718.086, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563a1e26f240] error while decoding MB 59 37, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 432, "h264_decode_error_count": 1332, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:40:35+09:00", "last_error_summary": "[h264 @ 0x563a1e26f240] error while decoding MB 59 37, bytestream -7", "last_exit_at": "2026-07-01T07:37:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.933, "last_started_at": "2026-07-01T07:37:51+09:00", "pid": 87289, "process": {"exists": false, "pid": 87289}, "restart_count": 472, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 473}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.457, "sample_count": 2006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1380652, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:40:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T07:40:35+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T07:40:35+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T07:40:35.439475+09:00", "mtime_epoch": 1782859235.4394753, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178741}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:40:35+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5023.781, "name": "celery", "pid": 267, "rss_mb": 5023.781, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.816, "pushed": 6.816, "read": 6.816}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.3, "count": 204, "max_ms": 42.876, "p95_ms": 36.245}, "gstreamer_push": {"avg_ms": 1.097, "count": 204, "max_ms": 1.293, "p95_ms": 1.17}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.179, "count": 204, "max_ms": 40.733, "p95_ms": 33.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.003, "count": 204, "max_ms": 85.927, "p95_ms": 82.986}, "yolo11_640_fire_11class": {"avg_ms": 49.534, "count": 204, "max_ms": 66.47, "p95_ms": 63.382}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.69, "count": 204, "max_ms": 82.331, "p95_ms": 77.714}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.761, "count": 204, "max_ms": 157.501, "p95_ms": 153.457}, "read_interval": {"avg_ms": 147.065, "count": 204, "max_ms": 169.23, "p95_ms": 166.45}, "triton_wait": {"avg_ms": 64.209, "count": 204, "max_ms": 85.939, "p95_ms": 82.998}, "visualize": {"avg_ms": 4.318, "count": 204, "max_ms": 5.523, "p95_ms": 5.026}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11505}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10717.637, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:40:35+09:00", "last_error_summary": "[h264 @ 0x563a1e26f240] error while decoding MB 59 37, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 432, "h264_decode_error_count": 1332, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:40:35+09:00", "last_error_summary": "[h264 @ 0x563a1e26f240] error while decoding MB 59 37, bytestream -7", "last_exit_at": "2026-07-01T07:40:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.015, "last_started_at": "2026-07-01T07:37:51+09:00", "pid": 87289, "process": {"exists": false, "pid": 87289}, "restart_count": 472, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 473}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.457, "sample_count": 2006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1380653, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:40:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:40:35+09:00", "last_frame_read_age_sec": 0.133, "last_frame_read_at": "2026-07-01T07:40:35+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-01T07:40:35.439475+09:00", "mtime_epoch": 1782859235.4394753, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178741}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:40:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5023.781, "name": "celery", "pid": 267, "rss_mb": 5023.781, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.831, "pushed": 6.831, "read": 6.798}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.292, "count": 204, "max_ms": 42.876, "p95_ms": 36.245}, "gstreamer_push": {"avg_ms": 1.098, "count": 204, "max_ms": 1.293, "p95_ms": 1.17}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.184, "count": 204, "max_ms": 40.733, "p95_ms": 33.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.021, "count": 204, "max_ms": 85.927, "p95_ms": 82.986}, "yolo11_640_fire_11class": {"avg_ms": 49.532, "count": 204, "max_ms": 66.47, "p95_ms": 63.382}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.703, "count": 204, "max_ms": 82.331, "p95_ms": 77.714}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.779, "count": 204, "max_ms": 157.501, "p95_ms": 153.457}, "read_interval": {"avg_ms": 147.127, "count": 203, "max_ms": 169.23, "p95_ms": 166.45}, "triton_wait": {"avg_ms": 64.227, "count": 204, "max_ms": 85.939, "p95_ms": 82.998}, "visualize": {"avg_ms": 4.319, "count": 204, "max_ms": 5.523, "p95_ms": 5.026}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.862, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11505}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10717.629, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 432, "h264_decode_error_count": 1332, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:40:35+09:00", "last_error_summary": "[h264 @ 0x563a1e26f240] error while decoding MB 59 37, bytestream -7", "last_exit_at": "2026-07-01T07:40:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.02, "last_started_at": "2026-07-01T07:37:51+09:00", "pid": 87289, "process": {"exists": false, "pid": 87289}, "restart_count": 472, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 473}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.457, "sample_count": 2006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1380653, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:40:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:40:35+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-01T07:40:35+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-01T07:40:35.439475+09:00", "mtime_epoch": 1782859235.4394753, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178741}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:40:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5023.781, "name": "celery", "pid": 267, "rss_mb": 5023.781, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.83, "pushed": 6.83, "read": 6.797}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.292, "count": 204, "max_ms": 42.876, "p95_ms": 36.245}, "gstreamer_push": {"avg_ms": 1.098, "count": 204, "max_ms": 1.293, "p95_ms": 1.17}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.184, "count": 204, "max_ms": 40.733, "p95_ms": 33.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.021, "count": 204, "max_ms": 85.927, "p95_ms": 82.986}, "yolo11_640_fire_11class": {"avg_ms": 49.532, "count": 204, "max_ms": 66.47, "p95_ms": 63.382}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.703, "count": 204, "max_ms": 82.331, "p95_ms": 77.714}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.779, "count": 204, "max_ms": 157.501, "p95_ms": 153.457}, "read_interval": {"avg_ms": 147.127, "count": 203, "max_ms": 169.23, "p95_ms": 166.45}, "triton_wait": {"avg_ms": 64.227, "count": 204, "max_ms": 85.939, "p95_ms": 82.998}, "visualize": {"avg_ms": 4.319, "count": 204, "max_ms": 5.523, "p95_ms": 5.026}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11505}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10826.188, "limit_mb": 8796093022207.996}, "detail": {"pid": 87396, "restart_count": 473, "start_count": 474}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 432, "h264_decode_error_count": 1332, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:40:35+09:00", "last_error_summary": "[h264 @ 0x563a1e26f240] error while decoding MB 59 37, bytestream -7", "last_exit_at": "2026-07-01T07:40:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:40:38+09:00", "pid": 87396, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.207, "name": "ffmpeg", "pid": 87396, "rss_mb": 139.207, "state": "S (sleeping)", "threads": 106}, "restart_count": 473, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 474}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.428, "sample_count": 1976, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1380653, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:40:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.539, "last_frame_pushed_at": "2026-07-01T07:40:35+09:00", "last_frame_read_age_sec": 4.668, "last_frame_read_at": "2026-07-01T07:40:35+09:00", "mjpeg": {"age_sec": 4.521, "exists": true, "mtime": "2026-07-01T07:40:35.579479+09:00", "mtime_epoch": 1782859235.5794785, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178877}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:40:40+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5023.781, "name": "celery", "pid": 267, "rss_mb": 5023.781, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 173, "pushed_frames": 173, "read_frames": 172, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.776, "pushed": 5.776, "read": 5.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.166, "count": 173, "max_ms": 42.876, "p95_ms": 36.012}, "gstreamer_push": {"avg_ms": 1.101, "count": 173, "max_ms": 1.293, "p95_ms": 1.172}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.364, "count": 173, "max_ms": 40.733, "p95_ms": 33.679}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.604, "count": 173, "max_ms": 85.927, "p95_ms": 83.27}, "yolo11_640_fire_11class": {"avg_ms": 50.006, "count": 173, "max_ms": 66.47, "p95_ms": 63.888}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.203, "count": 173, "max_ms": 82.331, "p95_ms": 77.973}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.324, "count": 173, "max_ms": 157.501, "p95_ms": 153.706}, "read_interval": {"avg_ms": 147.74, "count": 172, "max_ms": 169.23, "p95_ms": 167.037}, "triton_wait": {"avg_ms": 64.778, "count": 173, "max_ms": 85.939, "p95_ms": 83.282}, "visualize": {"avg_ms": 4.339, "count": 173, "max_ms": 5.523, "p95_ms": 5.065}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11514}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10720.258, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5639d573af00] error while decoding MB 14 36, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 433, "h264_decode_error_count": 1333, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:43:21+09:00", "last_error_summary": "[h264 @ 0x5639d573af00] error while decoding MB 14 36, bytestream -7", "last_exit_at": "2026-07-01T07:40:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.716, "last_started_at": "2026-07-01T07:40:38+09:00", "pid": 87396, "process": {"exists": false, "pid": 87396}, "restart_count": 473, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 474}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.089, "sample_count": 2006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1381755, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:43:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T07:43:21+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T07:43:21+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T07:43:21.774953+09:00", "mtime_epoch": 1782859401.7749527, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175175}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:43:21+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5024.297, "name": "celery", "pid": 267, "rss_mb": 5024.297, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.877, "pushed": 6.877, "read": 6.877}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.419, "count": 206, "max_ms": 36.644, "p95_ms": 35.157}, "gstreamer_push": {"avg_ms": 1.09, "count": 206, "max_ms": 1.268, "p95_ms": 1.196}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.758, "count": 206, "max_ms": 39.228, "p95_ms": 33.898}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.128, "count": 206, "max_ms": 85.741, "p95_ms": 82.265}, "yolo11_640_fire_11class": {"avg_ms": 49.011, "count": 206, "max_ms": 70.903, "p95_ms": 62.691}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.907, "count": 206, "max_ms": 79.502, "p95_ms": 76.581}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.752, "count": 206, "max_ms": 158.433, "p95_ms": 153.224}, "read_interval": {"avg_ms": 145.896, "count": 206, "max_ms": 170.499, "p95_ms": 165.577}, "triton_wait": {"avg_ms": 63.19, "count": 206, "max_ms": 85.754, "p95_ms": 82.28}, "visualize": {"avg_ms": 4.269, "count": 206, "max_ms": 5.206, "p95_ms": 4.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11514}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10719.594, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:43:21+09:00", "last_error_summary": "[h264 @ 0x5639d573af00] error while decoding MB 14 36, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 433, "h264_decode_error_count": 1333, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:43:21+09:00", "last_error_summary": "[h264 @ 0x5639d573af00] error while decoding MB 14 36, bytestream -7", "last_exit_at": "2026-07-01T07:43:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.826, "last_started_at": "2026-07-01T07:40:38+09:00", "pid": 87396, "process": {"exists": false, "pid": 87396}, "restart_count": 473, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 474}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.089, "sample_count": 2007, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1381756, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:43:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T07:43:21+09:00", "last_frame_read_age_sec": 0.161, "last_frame_read_at": "2026-07-01T07:43:21+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-01T07:43:21.774953+09:00", "mtime_epoch": 1782859401.7749527, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175175}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:43:21+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5024.297, "name": "celery", "pid": 267, "rss_mb": 5024.297, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.883, "pushed": 6.883, "read": 6.85}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.413, "count": 206, "max_ms": 36.644, "p95_ms": 35.157}, "gstreamer_push": {"avg_ms": 1.09, "count": 206, "max_ms": 1.268, "p95_ms": 1.196}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.759, "count": 206, "max_ms": 39.228, "p95_ms": 33.898}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.138, "count": 206, "max_ms": 85.741, "p95_ms": 82.444}, "yolo11_640_fire_11class": {"avg_ms": 49.023, "count": 206, "max_ms": 70.903, "p95_ms": 63.044}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.92, "count": 206, "max_ms": 79.502, "p95_ms": 77.079}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.764, "count": 206, "max_ms": 158.433, "p95_ms": 153.224}, "read_interval": {"avg_ms": 145.82, "count": 205, "max_ms": 170.499, "p95_ms": 165.577}, "triton_wait": {"avg_ms": 63.199, "count": 206, "max_ms": 85.754, "p95_ms": 82.456}, "visualize": {"avg_ms": 4.27, "count": 206, "max_ms": 5.206, "p95_ms": 4.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11514}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10719.586, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 433, "h264_decode_error_count": 1333, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:43:21+09:00", "last_error_summary": "[h264 @ 0x5639d573af00] error while decoding MB 14 36, bytestream -7", "last_exit_at": "2026-07-01T07:43:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.831, "last_started_at": "2026-07-01T07:40:38+09:00", "pid": 87396, "process": {"exists": false, "pid": 87396}, "restart_count": 473, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 474}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.089, "sample_count": 2006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1381756, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:43:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T07:43:21+09:00", "last_frame_read_age_sec": 0.165, "last_frame_read_at": "2026-07-01T07:43:21+09:00", "mjpeg": {"age_sec": 0.155, "exists": true, "mtime": "2026-07-01T07:43:21.774953+09:00", "mtime_epoch": 1782859401.7749527, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175175}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:43:21+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5024.297, "name": "celery", "pid": 267, "rss_mb": 5024.297, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.882, "pushed": 6.882, "read": 6.849}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.413, "count": 206, "max_ms": 36.644, "p95_ms": 35.157}, "gstreamer_push": {"avg_ms": 1.09, "count": 206, "max_ms": 1.268, "p95_ms": 1.196}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.759, "count": 206, "max_ms": 39.228, "p95_ms": 33.898}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.138, "count": 206, "max_ms": 85.741, "p95_ms": 82.444}, "yolo11_640_fire_11class": {"avg_ms": 49.023, "count": 206, "max_ms": 70.903, "p95_ms": 63.044}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.92, "count": 206, "max_ms": 79.502, "p95_ms": 77.079}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.764, "count": 206, "max_ms": 158.433, "p95_ms": 153.224}, "read_interval": {"avg_ms": 145.82, "count": 205, "max_ms": 170.499, "p95_ms": 165.577}, "triton_wait": {"avg_ms": 63.199, "count": 206, "max_ms": 85.754, "p95_ms": 82.456}, "visualize": {"avg_ms": 4.27, "count": 206, "max_ms": 5.206, "p95_ms": 4.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11514}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10827.508, "limit_mb": 8796093022207.996}, "detail": {"pid": 87503, "restart_count": 474, "start_count": 475}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 433, "h264_decode_error_count": 1333, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:43:21+09:00", "last_error_summary": "[h264 @ 0x5639d573af00] error while decoding MB 14 36, bytestream -7", "last_exit_at": "2026-07-01T07:43:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:43:24+09:00", "pid": 87503, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.773, "name": "ffmpeg", "pid": 87503, "rss_mb": 138.773, "state": "S (sleeping)", "threads": 106}, "restart_count": 474, "restart_count_per_hour": 23, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 475}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.054, "sample_count": 1977, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1381756, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:43:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.431, "last_frame_pushed_at": "2026-07-01T07:43:21+09:00", "last_frame_read_age_sec": 4.585, "last_frame_read_at": "2026-07-01T07:43:21+09:00", "mjpeg": {"age_sec": 4.415, "exists": true, "mtime": "2026-07-01T07:43:21.934956+09:00", "mtime_epoch": 1782859401.934956, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175676}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:43:26+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5024.297, "name": "celery", "pid": 267, "rss_mb": 5024.297, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 176, "pushed_frames": 176, "read_frames": 175, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.889, "pushed": 5.889, "read": 5.856}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.334, "count": 176, "max_ms": 36.644, "p95_ms": 35.147}, "gstreamer_push": {"avg_ms": 1.086, "count": 176, "max_ms": 1.268, "p95_ms": 1.196}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.599, "count": 176, "max_ms": 39.228, "p95_ms": 33.898}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.873, "count": 176, "max_ms": 85.741, "p95_ms": 82.933}, "yolo11_640_fire_11class": {"avg_ms": 48.947, "count": 176, "max_ms": 70.903, "p95_ms": 63.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.82, "count": 176, "max_ms": 79.502, "p95_ms": 77.976}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.375, "count": 176, "max_ms": 158.433, "p95_ms": 153.575}, "read_interval": {"avg_ms": 145.392, "count": 175, "max_ms": 170.499, "p95_ms": 165.972}, "triton_wait": {"avg_ms": 62.943, "count": 176, "max_ms": 85.754, "p95_ms": 82.945}, "visualize": {"avg_ms": 4.27, "count": 176, "max_ms": 4.986, "p95_ms": 4.64}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11523}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10721.762, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558ec85f6c40] error while decoding MB 67 43, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 434, "h264_decode_error_count": 1334, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:46:07+09:00", "last_error_summary": "[h264 @ 0x558ec85f6c40] error while decoding MB 67 43, bytestream -5", "last_exit_at": "2026-07-01T07:43:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.416, "last_started_at": "2026-07-01T07:43:24+09:00", "pid": 87503, "process": {"exists": false, "pid": 87503}, "restart_count": 474, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 475}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.678, "sample_count": 1998, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1382837, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:46:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T07:46:07+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T07:46:07+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T07:46:07.722284+09:00", "mtime_epoch": 1782859567.7222838, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175950}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:46:07+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5024.812, "name": "celery", "pid": 267, "rss_mb": 5024.812, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 199, "pushed_frames": 199, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.662, "pushed": 6.662, "read": 6.662}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.524, "count": 199, "max_ms": 39.627, "p95_ms": 35.946}, "gstreamer_push": {"avg_ms": 1.107, "count": 199, "max_ms": 1.352, "p95_ms": 1.175}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.86, "count": 199, "max_ms": 41.231, "p95_ms": 36.2}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.307, "count": 199, "max_ms": 93.071, "p95_ms": 85.27}, "yolo11_640_fire_11class": {"avg_ms": 50.773, "count": 199, "max_ms": 73.672, "p95_ms": 64.137}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.3, "count": 199, "max_ms": 83.751, "p95_ms": 80.482}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.311, "count": 199, "max_ms": 166.89, "p95_ms": 157.309}, "read_interval": {"avg_ms": 150.54, "count": 199, "max_ms": 177.514, "p95_ms": 169.955}, "triton_wait": {"avg_ms": 67.621, "count": 199, "max_ms": 93.083, "p95_ms": 85.282}, "visualize": {"avg_ms": 4.333, "count": 199, "max_ms": 5.278, "p95_ms": 4.883}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.872, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11523}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10721.227, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:46:07+09:00", "last_error_summary": "[h264 @ 0x558ec85f6c40] error while decoding MB 67 43, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 434, "h264_decode_error_count": 1334, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:46:07+09:00", "last_error_summary": "[h264 @ 0x558ec85f6c40] error while decoding MB 67 43, bytestream -5", "last_exit_at": "2026-07-01T07:46:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.508, "last_started_at": "2026-07-01T07:43:24+09:00", "pid": 87503, "process": {"exists": false, "pid": 87503}, "restart_count": 474, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 475}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.678, "sample_count": 1998, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1382838, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:46:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:46:07+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T07:46:07+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T07:46:07.722284+09:00", "mtime_epoch": 1782859567.7222838, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175950}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:46:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5024.812, "name": "celery", "pid": 267, "rss_mb": 5024.812, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.675, "pushed": 6.675, "read": 6.642}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.538, "count": 200, "max_ms": 39.627, "p95_ms": 35.794}, "gstreamer_push": {"avg_ms": 1.107, "count": 200, "max_ms": 1.352, "p95_ms": 1.172}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.853, "count": 200, "max_ms": 41.231, "p95_ms": 35.696}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.28, "count": 200, "max_ms": 93.071, "p95_ms": 85.261}, "yolo11_640_fire_11class": {"avg_ms": 50.772, "count": 200, "max_ms": 73.672, "p95_ms": 64.005}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.272, "count": 200, "max_ms": 83.751, "p95_ms": 80.352}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.298, "count": 200, "max_ms": 166.89, "p95_ms": 156.336}, "read_interval": {"avg_ms": 150.54, "count": 199, "max_ms": 177.514, "p95_ms": 169.955}, "triton_wait": {"avg_ms": 67.592, "count": 200, "max_ms": 93.083, "p95_ms": 85.273}, "visualize": {"avg_ms": 4.334, "count": 200, "max_ms": 5.278, "p95_ms": 4.838}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11523}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10721.309, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 434, "h264_decode_error_count": 1334, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:46:07+09:00", "last_error_summary": "[h264 @ 0x558ec85f6c40] error while decoding MB 67 43, bytestream -5", "last_exit_at": "2026-07-01T07:46:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.513, "last_started_at": "2026-07-01T07:43:24+09:00", "pid": 87503, "process": {"exists": false, "pid": 87503}, "restart_count": 474, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 475}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.678, "sample_count": 1998, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1382838, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:46:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:46:07+09:00", "last_frame_read_age_sec": 0.147, "last_frame_read_at": "2026-07-01T07:46:07+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-01T07:46:07.722284+09:00", "mtime_epoch": 1782859567.7222838, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175950}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:46:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5024.812, "name": "celery", "pid": 267, "rss_mb": 5024.812, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.674, "pushed": 6.674, "read": 6.641}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.522, "count": 199, "max_ms": 39.627, "p95_ms": 35.946}, "gstreamer_push": {"avg_ms": 1.107, "count": 200, "max_ms": 1.352, "p95_ms": 1.172}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.863, "count": 199, "max_ms": 41.231, "p95_ms": 36.2}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.299, "count": 199, "max_ms": 93.071, "p95_ms": 85.27}, "yolo11_640_fire_11class": {"avg_ms": 50.773, "count": 199, "max_ms": 73.672, "p95_ms": 64.137}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.29, "count": 199, "max_ms": 83.751, "p95_ms": 80.482}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.298, "count": 200, "max_ms": 166.89, "p95_ms": 156.336}, "read_interval": {"avg_ms": 150.54, "count": 199, "max_ms": 177.514, "p95_ms": 169.955}, "triton_wait": {"avg_ms": 67.613, "count": 199, "max_ms": 93.083, "p95_ms": 85.282}, "visualize": {"avg_ms": 4.333, "count": 199, "max_ms": 5.278, "p95_ms": 4.883}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11523}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10834.773, "limit_mb": 8796093022207.996}, "detail": {"pid": 87610, "restart_count": 475, "start_count": 476}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 434, "h264_decode_error_count": 1334, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:46:07+09:00", "last_error_summary": "[h264 @ 0x558ec85f6c40] error while decoding MB 67 43, bytestream -5", "last_exit_at": "2026-07-01T07:46:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:46:10+09:00", "pid": 87610, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.48, "name": "ffmpeg", "pid": 87610, "rss_mb": 144.48, "state": "S (sleeping)", "threads": 106}, "restart_count": 475, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 476}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.695, "sample_count": 1968, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1382838, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:46:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.378, "last_frame_pushed_at": "2026-07-01T07:46:07+09:00", "last_frame_read_age_sec": 4.515, "last_frame_read_at": "2026-07-01T07:46:07+09:00", "mjpeg": {"age_sec": 4.36, "exists": true, "mtime": "2026-07-01T07:46:07.870287+09:00", "mtime_epoch": 1782859567.8702867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176521}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:46:12+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5024.812, "name": "celery", "pid": 267, "rss_mb": 5024.812, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 171, "pushed_frames": 171, "read_frames": 170, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.719, "pushed": 5.719, "read": 5.686}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.601, "count": 171, "max_ms": 39.627, "p95_ms": 35.993}, "gstreamer_push": {"avg_ms": 1.106, "count": 171, "max_ms": 1.352, "p95_ms": 1.175}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.966, "count": 171, "max_ms": 41.193, "p95_ms": 36.2}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.096, "count": 171, "max_ms": 93.071, "p95_ms": 85.27}, "yolo11_640_fire_11class": {"avg_ms": 50.579, "count": 171, "max_ms": 73.672, "p95_ms": 64.258}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.985, "count": 171, "max_ms": 83.751, "p95_ms": 80.482}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.954, "count": 171, "max_ms": 161.97, "p95_ms": 156.336}, "read_interval": {"avg_ms": 150.22, "count": 170, "max_ms": 174.115, "p95_ms": 169.653}, "triton_wait": {"avg_ms": 67.43, "count": 171, "max_ms": 93.083, "p95_ms": 85.282}, "visualize": {"avg_ms": 4.324, "count": 171, "max_ms": 5.278, "p95_ms": 4.883}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11532}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10722.605, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:46:07+09:00", "last_error_summary": "[h264 @ 0x558ec85f6c40] error while decoding MB 67 43, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 434, "h264_decode_error_count": 1334, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:46:07+09:00", "last_error_summary": "[h264 @ 0x558ec85f6c40] error while decoding MB 67 43, bytestream -5", "last_exit_at": "2026-07-01T07:48:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.354, "last_started_at": "2026-07-01T07:46:10+09:00", "pid": 87610, "process": {"exists": false, "pid": 87610}, "restart_count": 475, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 476}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.597, "sample_count": 1991, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1383933, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:48:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:48:53+09:00", "last_frame_read_age_sec": 0.146, "last_frame_read_at": "2026-07-01T07:48:53+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-01T07:48:53.445511+09:00", "mtime_epoch": 1782859733.445511, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180548}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:48:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5025.586, "name": "celery", "pid": 267, "rss_mb": 5025.586, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.816, "pushed": 6.816, "read": 6.783}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.742, "count": 204, "max_ms": 38.477, "p95_ms": 36.018}, "gstreamer_push": {"avg_ms": 1.098, "count": 204, "max_ms": 1.508, "p95_ms": 1.26}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.888, "count": 204, "max_ms": 41.918, "p95_ms": 35.728}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.068, "count": 204, "max_ms": 86.753, "p95_ms": 83.423}, "yolo11_640_fire_11class": {"avg_ms": 49.441, "count": 204, "max_ms": 65.763, "p95_ms": 64.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.124, "count": 204, "max_ms": 82.448, "p95_ms": 79.387}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.013, "count": 204, "max_ms": 161.475, "p95_ms": 152.775}, "read_interval": {"avg_ms": 147.44, "count": 203, "max_ms": 173.542, "p95_ms": 166.358}, "triton_wait": {"avg_ms": 65.257, "count": 204, "max_ms": 86.766, "p95_ms": 83.434}, "visualize": {"avg_ms": 4.242, "count": 204, "max_ms": 5.191, "p95_ms": 4.698}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11532}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10722.598, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 434, "h264_decode_error_count": 1334, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:46:07+09:00", "last_error_summary": "[h264 @ 0x558ec85f6c40] error while decoding MB 67 43, bytestream -5", "last_exit_at": "2026-07-01T07:48:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.359, "last_started_at": "2026-07-01T07:46:10+09:00", "pid": 87610, "process": {"exists": false, "pid": 87610}, "restart_count": 475, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 476}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.597, "sample_count": 1991, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1383933, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:48:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:48:53+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-01T07:48:53+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-01T07:48:53.445511+09:00", "mtime_epoch": 1782859733.445511, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180548}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:48:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5025.586, "name": "celery", "pid": 267, "rss_mb": 5025.586, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.815, "pushed": 6.815, "read": 6.782}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.742, "count": 204, "max_ms": 38.477, "p95_ms": 36.018}, "gstreamer_push": {"avg_ms": 1.098, "count": 204, "max_ms": 1.508, "p95_ms": 1.26}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.888, "count": 204, "max_ms": 41.918, "p95_ms": 35.728}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.068, "count": 204, "max_ms": 86.753, "p95_ms": 83.423}, "yolo11_640_fire_11class": {"avg_ms": 49.441, "count": 204, "max_ms": 65.763, "p95_ms": 64.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.124, "count": 204, "max_ms": 82.448, "p95_ms": 79.387}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.013, "count": 204, "max_ms": 161.475, "p95_ms": 152.775}, "read_interval": {"avg_ms": 147.44, "count": 203, "max_ms": 173.542, "p95_ms": 166.358}, "triton_wait": {"avg_ms": 65.257, "count": 204, "max_ms": 86.766, "p95_ms": 83.434}, "visualize": {"avg_ms": 4.242, "count": 204, "max_ms": 5.191, "p95_ms": 4.698}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11532}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10830.504, "limit_mb": 8796093022207.996}, "detail": {"pid": 87717, "restart_count": 476, "start_count": 477}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 434, "h264_decode_error_count": 1334, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:46:07+09:00", "last_error_summary": "[h264 @ 0x558ec85f6c40] error while decoding MB 67 43, bytestream -5", "last_exit_at": "2026-07-01T07:48:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:48:56+09:00", "pid": 87717, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.359, "name": "ffmpeg", "pid": 87717, "rss_mb": 139.359, "state": "S (sleeping)", "threads": 106}, "restart_count": 476, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 477}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.639, "sample_count": 1960, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1383933, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:48:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.602, "last_frame_pushed_at": "2026-07-01T07:48:53+09:00", "last_frame_read_age_sec": 4.742, "last_frame_read_at": "2026-07-01T07:48:53+09:00", "mjpeg": {"age_sec": 4.587, "exists": true, "mtime": "2026-07-01T07:48:53.593514+09:00", "mtime_epoch": 1782859733.593514, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180554}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:48:58+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5025.586, "name": "celery", "pid": 267, "rss_mb": 5025.586, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.813, "pushed": 5.813, "read": 5.78}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.7, "count": 174, "max_ms": 38.477, "p95_ms": 36.098}, "gstreamer_push": {"avg_ms": 1.096, "count": 174, "max_ms": 1.508, "p95_ms": 1.26}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.819, "count": 174, "max_ms": 41.918, "p95_ms": 36.961}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.179, "count": 174, "max_ms": 85.491, "p95_ms": 82.901}, "yolo11_640_fire_11class": {"avg_ms": 48.762, "count": 174, "max_ms": 65.763, "p95_ms": 64.162}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.858, "count": 174, "max_ms": 82.448, "p95_ms": 77.557}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.086, "count": 174, "max_ms": 161.475, "p95_ms": 151.807}, "read_interval": {"avg_ms": 146.457, "count": 173, "max_ms": 173.542, "p95_ms": 166.052}, "triton_wait": {"avg_ms": 64.357, "count": 174, "max_ms": 85.504, "p95_ms": 82.914}, "visualize": {"avg_ms": 4.235, "count": 174, "max_ms": 5.191, "p95_ms": 4.698}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11541}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10724.906, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5625a2b9fa40] error while decoding MB 30 38, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 435, "h264_decode_error_count": 1335, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:51:39+09:00", "last_error_summary": "[h264 @ 0x5625a2b9fa40] error while decoding MB 30 38, bytestream -9", "last_exit_at": "2026-07-01T07:48:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.594, "last_started_at": "2026-07-01T07:48:56+09:00", "pid": 87717, "process": {"exists": false, "pid": 87717}, "restart_count": 476, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 477}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.359, "sample_count": 2008, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1385031, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:51:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.053, "last_frame_pushed_at": "2026-07-01T07:51:39+09:00", "last_frame_read_age_sec": 0.042, "last_frame_read_at": "2026-07-01T07:51:39+09:00", "mjpeg": {"age_sec": 0.036, "exists": true, "mtime": "2026-07-01T07:51:39.736677+09:00", "mtime_epoch": 1782859899.7366772, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179908}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:51:39+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5026.102, "name": "celery", "pid": 267, "rss_mb": 5026.102, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.739, "pushed": 6.739, "read": 6.739}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.592, "count": 201, "max_ms": 39.813, "p95_ms": 35.936}, "gstreamer_push": {"avg_ms": 1.091, "count": 202, "max_ms": 1.203, "p95_ms": 1.156}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.616, "count": 201, "max_ms": 41.228, "p95_ms": 33.808}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.604, "count": 201, "max_ms": 87.222, "p95_ms": 84.775}, "yolo11_640_fire_11class": {"avg_ms": 49.917, "count": 201, "max_ms": 66.313, "p95_ms": 64.069}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.928, "count": 201, "max_ms": 80.222, "p95_ms": 78.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.25, "count": 202, "max_ms": 157.438, "p95_ms": 154.766}, "read_interval": {"avg_ms": 148.913, "count": 202, "max_ms": 171.643, "p95_ms": 168.684}, "triton_wait": {"avg_ms": 66.767, "count": 201, "max_ms": 87.235, "p95_ms": 84.788}, "visualize": {"avg_ms": 4.204, "count": 201, "max_ms": 5.24, "p95_ms": 4.532}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11541}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10724.047, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:51:39+09:00", "last_error_summary": "[h264 @ 0x5625a2b9fa40] error while decoding MB 30 38, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 435, "h264_decode_error_count": 1335, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:51:39+09:00", "last_error_summary": "[h264 @ 0x5625a2b9fa40] error while decoding MB 30 38, bytestream -9", "last_exit_at": "2026-07-01T07:51:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.682, "last_started_at": "2026-07-01T07:48:56+09:00", "pid": 87717, "process": {"exists": false, "pid": 87717}, "restart_count": 476, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 477}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.359, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1385032, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:51:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:51:39+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-01T07:51:39+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-01T07:51:39.736677+09:00", "mtime_epoch": 1782859899.7366772, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179908}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:51:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5026.102, "name": "celery", "pid": 267, "rss_mb": 5026.102, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.751, "pushed": 6.751, "read": 6.718}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.593, "count": 202, "max_ms": 39.813, "p95_ms": 35.936}, "gstreamer_push": {"avg_ms": 1.091, "count": 202, "max_ms": 1.203, "p95_ms": 1.156}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.612, "count": 202, "max_ms": 41.228, "p95_ms": 33.808}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.55, "count": 202, "max_ms": 87.222, "p95_ms": 84.775}, "yolo11_640_fire_11class": {"avg_ms": 49.862, "count": 202, "max_ms": 66.313, "p95_ms": 64.069}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.872, "count": 202, "max_ms": 80.222, "p95_ms": 78.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.123, "count": 202, "max_ms": 157.438, "p95_ms": 154.766}, "read_interval": {"avg_ms": 148.84, "count": 201, "max_ms": 171.643, "p95_ms": 168.684}, "triton_wait": {"avg_ms": 66.712, "count": 202, "max_ms": 87.235, "p95_ms": 84.788}, "visualize": {"avg_ms": 4.203, "count": 202, "max_ms": 5.24, "p95_ms": 4.532}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11541}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10723.984, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 435, "h264_decode_error_count": 1335, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:51:39+09:00", "last_error_summary": "[h264 @ 0x5625a2b9fa40] error while decoding MB 30 38, bytestream -9", "last_exit_at": "2026-07-01T07:51:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.687, "last_started_at": "2026-07-01T07:48:56+09:00", "pid": 87717, "process": {"exists": false, "pid": 87717}, "restart_count": 476, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 477}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.359, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1385032, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:51:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T07:51:39+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-01T07:51:39+09:00", "mjpeg": {"age_sec": 0.129, "exists": true, "mtime": "2026-07-01T07:51:39.736677+09:00", "mtime_epoch": 1782859899.7366772, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179908}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:51:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5026.102, "name": "celery", "pid": 267, "rss_mb": 5026.102, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.75, "pushed": 6.75, "read": 6.717}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.593, "count": 202, "max_ms": 39.813, "p95_ms": 35.936}, "gstreamer_push": {"avg_ms": 1.091, "count": 202, "max_ms": 1.203, "p95_ms": 1.156}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.612, "count": 202, "max_ms": 41.228, "p95_ms": 33.808}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.55, "count": 202, "max_ms": 87.222, "p95_ms": 84.775}, "yolo11_640_fire_11class": {"avg_ms": 49.862, "count": 202, "max_ms": 66.313, "p95_ms": 64.069}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.872, "count": 202, "max_ms": 80.222, "p95_ms": 78.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.123, "count": 202, "max_ms": 157.438, "p95_ms": 154.766}, "read_interval": {"avg_ms": 148.84, "count": 201, "max_ms": 171.643, "p95_ms": 168.684}, "triton_wait": {"avg_ms": 66.712, "count": 202, "max_ms": 87.235, "p95_ms": 84.788}, "visualize": {"avg_ms": 4.203, "count": 202, "max_ms": 5.24, "p95_ms": 4.532}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11541}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10832.16, "limit_mb": 8796093022207.996}, "detail": {"pid": 87824, "restart_count": 477, "start_count": 478}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 435, "h264_decode_error_count": 1335, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:51:39+09:00", "last_error_summary": "[h264 @ 0x5625a2b9fa40] error while decoding MB 30 38, bytestream -9", "last_exit_at": "2026-07-01T07:51:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:51:42+09:00", "pid": 87824, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.246, "name": "ffmpeg", "pid": 87824, "rss_mb": 139.246, "state": "S (sleeping)", "threads": 106}, "restart_count": 477, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 478}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.207, "sample_count": 1977, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1385032, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:51:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.802, "last_frame_pushed_at": "2026-07-01T07:51:39+09:00", "last_frame_read_age_sec": 4.929, "last_frame_read_at": "2026-07-01T07:51:39+09:00", "mjpeg": {"age_sec": 4.787, "exists": true, "mtime": "2026-07-01T07:51:39.872680+09:00", "mtime_epoch": 1782859899.8726797, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179964}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:51:44+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5026.102, "name": "celery", "pid": 267, "rss_mb": 5026.102, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 171, "pushed_frames": 171, "read_frames": 170, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.709, "pushed": 5.709, "read": 5.676}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.632, "count": 171, "max_ms": 39.813, "p95_ms": 36.149}, "gstreamer_push": {"avg_ms": 1.088, "count": 171, "max_ms": 1.187, "p95_ms": 1.155}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.589, "count": 171, "max_ms": 41.228, "p95_ms": 34.107}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.27, "count": 171, "max_ms": 87.222, "p95_ms": 84.348}, "yolo11_640_fire_11class": {"avg_ms": 49.362, "count": 171, "max_ms": 66.313, "p95_ms": 63.579}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.627, "count": 171, "max_ms": 80.222, "p95_ms": 78.463}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.661, "count": 171, "max_ms": 157.394, "p95_ms": 153.815}, "read_interval": {"avg_ms": 148.095, "count": 170, "max_ms": 171.321, "p95_ms": 166.664}, "triton_wait": {"avg_ms": 66.401, "count": 171, "max_ms": 87.235, "p95_ms": 84.36}, "visualize": {"avg_ms": 4.184, "count": 171, "max_ms": 5.24, "p95_ms": 4.553}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11550}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10725.426, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559d4ecba100] error while decoding MB 15 35, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 436, "h264_decode_error_count": 1336, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:54:26+09:00", "last_error_summary": "[h264 @ 0x559d4ecba100] error while decoding MB 15 35, bytestream -5", "last_exit_at": "2026-07-01T07:51:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.408, "last_started_at": "2026-07-01T07:51:42+09:00", "pid": 87824, "process": {"exists": false, "pid": 87824}, "restart_count": 477, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 478}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.498, "sample_count": 1993, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1386116, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:54:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T07:54:26+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T07:54:26+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T07:54:26.023791+09:00", "mtime_epoch": 1782860066.0237908, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179348}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:54:26+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5026.617, "name": "celery", "pid": 267, "rss_mb": 5026.617, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.707, "pushed": 6.707, "read": 6.707}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.404, "count": 200, "max_ms": 41.086, "p95_ms": 35.971}, "gstreamer_push": {"avg_ms": 1.096, "count": 201, "max_ms": 1.477, "p95_ms": 1.155}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.965, "count": 200, "max_ms": 41.75, "p95_ms": 36.755}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.275, "count": 200, "max_ms": 90.689, "p95_ms": 84.083}, "yolo11_640_fire_11class": {"avg_ms": 50.244, "count": 200, "max_ms": 67.241, "p95_ms": 64.026}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.068, "count": 200, "max_ms": 83.91, "p95_ms": 79.886}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.359, "count": 201, "max_ms": 180.772, "p95_ms": 160.193}, "read_interval": {"avg_ms": 149.491, "count": 201, "max_ms": 193.258, "p95_ms": 171.003}, "triton_wait": {"avg_ms": 65.556, "count": 200, "max_ms": 90.702, "p95_ms": 84.096}, "visualize": {"avg_ms": 4.328, "count": 200, "max_ms": 6.519, "p95_ms": 5.171}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11550}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10725.09, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:54:26+09:00", "last_error_summary": "[h264 @ 0x559d4ecba100] error while decoding MB 15 35, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 436, "h264_decode_error_count": 1336, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:54:26+09:00", "last_error_summary": "[h264 @ 0x559d4ecba100] error while decoding MB 15 35, bytestream -5", "last_exit_at": "2026-07-01T07:54:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.508, "last_started_at": "2026-07-01T07:51:42+09:00", "pid": 87824, "process": {"exists": false, "pid": 87824}, "restart_count": 477, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 478}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.498, "sample_count": 1993, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1386117, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:54:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T07:54:26+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-01T07:54:26+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-01T07:54:26.023791+09:00", "mtime_epoch": 1782860066.0237908, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179348}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:54:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5026.617, "name": "celery", "pid": 267, "rss_mb": 5026.617, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.723, "pushed": 6.723, "read": 6.689}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.453, "count": 201, "max_ms": 42.261, "p95_ms": 36.064}, "gstreamer_push": {"avg_ms": 1.096, "count": 201, "max_ms": 1.477, "p95_ms": 1.156}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.958, "count": 201, "max_ms": 41.75, "p95_ms": 36.755}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.224, "count": 201, "max_ms": 90.689, "p95_ms": 84.083}, "yolo11_640_fire_11class": {"avg_ms": 50.212, "count": 201, "max_ms": 67.241, "p95_ms": 64.026}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.02, "count": 201, "max_ms": 83.91, "p95_ms": 79.886}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.438, "count": 201, "max_ms": 180.772, "p95_ms": 160.193}, "read_interval": {"avg_ms": 149.54, "count": 200, "max_ms": 193.258, "p95_ms": 171.003}, "triton_wait": {"avg_ms": 65.504, "count": 201, "max_ms": 90.702, "p95_ms": 84.096}, "visualize": {"avg_ms": 4.332, "count": 201, "max_ms": 6.519, "p95_ms": 5.171}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11550}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10725.203, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 436, "h264_decode_error_count": 1336, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:54:26+09:00", "last_error_summary": "[h264 @ 0x559d4ecba100] error while decoding MB 15 35, bytestream -5", "last_exit_at": "2026-07-01T07:54:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.514, "last_started_at": "2026-07-01T07:51:42+09:00", "pid": 87824, "process": {"exists": false, "pid": 87824}, "restart_count": 477, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 478}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.498, "sample_count": 1993, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1386117, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:54:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T07:54:26+09:00", "last_frame_read_age_sec": 0.157, "last_frame_read_at": "2026-07-01T07:54:26+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-01T07:54:26.023791+09:00", "mtime_epoch": 1782860066.0237908, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179348}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:54:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5026.617, "name": "celery", "pid": 267, "rss_mb": 5026.617, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.721, "pushed": 6.721, "read": 6.688}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.453, "count": 201, "max_ms": 42.261, "p95_ms": 36.064}, "gstreamer_push": {"avg_ms": 1.096, "count": 201, "max_ms": 1.477, "p95_ms": 1.156}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.958, "count": 201, "max_ms": 41.75, "p95_ms": 36.755}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.224, "count": 201, "max_ms": 90.689, "p95_ms": 84.083}, "yolo11_640_fire_11class": {"avg_ms": 50.212, "count": 201, "max_ms": 67.241, "p95_ms": 64.026}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.02, "count": 201, "max_ms": 83.91, "p95_ms": 79.886}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.438, "count": 201, "max_ms": 180.772, "p95_ms": 160.193}, "read_interval": {"avg_ms": 149.54, "count": 200, "max_ms": 193.258, "p95_ms": 171.003}, "triton_wait": {"avg_ms": 65.504, "count": 201, "max_ms": 90.702, "p95_ms": 84.096}, "visualize": {"avg_ms": 4.332, "count": 201, "max_ms": 6.519, "p95_ms": 5.171}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11550}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10830.812, "limit_mb": 8796093022207.996}, "detail": {"pid": 87931, "restart_count": 478, "start_count": 479}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 436, "h264_decode_error_count": 1336, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:54:26+09:00", "last_error_summary": "[h264 @ 0x559d4ecba100] error while decoding MB 15 35, bytestream -5", "last_exit_at": "2026-07-01T07:54:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:54:28+09:00", "pid": 87931, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.617, "name": "ffmpeg", "pid": 87931, "rss_mb": 136.617, "state": "S (sleeping)", "threads": 106}, "restart_count": 478, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 479}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.622, "sample_count": 1961, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1386117, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:54:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.78, "last_frame_pushed_at": "2026-07-01T07:54:26+09:00", "last_frame_read_age_sec": 4.925, "last_frame_read_at": "2026-07-01T07:54:26+09:00", "mjpeg": {"age_sec": 4.761, "exists": true, "mtime": "2026-07-01T07:54:26.179794+09:00", "mtime_epoch": 1782860066.1797938, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178776}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:54:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5026.617, "name": "celery", "pid": 267, "rss_mb": 5026.617, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 168, "pushed_frames": 168, "read_frames": 167, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.615, "pushed": 5.615, "read": 5.582}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.483, "count": 168, "max_ms": 42.261, "p95_ms": 36.064}, "gstreamer_push": {"avg_ms": 1.093, "count": 168, "max_ms": 1.477, "p95_ms": 1.149}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.978, "count": 168, "max_ms": 41.75, "p95_ms": 36.755}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.252, "count": 168, "max_ms": 89.195, "p95_ms": 84.083}, "yolo11_640_fire_11class": {"avg_ms": 50.336, "count": 168, "max_ms": 65.731, "p95_ms": 64.063}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.211, "count": 168, "max_ms": 83.91, "p95_ms": 80.044}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.526, "count": 168, "max_ms": 180.772, "p95_ms": 160.193}, "read_interval": {"avg_ms": 150.552, "count": 167, "max_ms": 193.258, "p95_ms": 171.003}, "triton_wait": {"avg_ms": 65.584, "count": 168, "max_ms": 89.209, "p95_ms": 84.096}, "visualize": {"avg_ms": 4.375, "count": 168, "max_ms": 6.519, "p95_ms": 5.185}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11560}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10727.73, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5627a0594f80] error while decoding MB 50 26, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 437, "h264_decode_error_count": 1337, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:57:12+09:00", "last_error_summary": "[h264 @ 0x5627a0594f80] error while decoding MB 50 26, bytestream -7", "last_exit_at": "2026-07-01T07:54:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.739, "last_started_at": "2026-07-01T07:54:28+09:00", "pid": 87931, "process": {"exists": false, "pid": 87931}, "restart_count": 478, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 479}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.843, "sample_count": 1985, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1387204, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:57:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T07:57:12+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T07:57:12+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T07:57:12.634873+09:00", "mtime_epoch": 1782860232.6348727, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180865}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:57:12+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5027.133, "name": "celery", "pid": 267, "rss_mb": 5027.133, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.701, "pushed": 6.701, "read": 6.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.842, "count": 201, "max_ms": 43.121, "p95_ms": 36.952}, "gstreamer_push": {"avg_ms": 1.109, "count": 201, "max_ms": 1.492, "p95_ms": 1.159}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.531, "count": 201, "max_ms": 43.234, "p95_ms": 34.854}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.517, "count": 201, "max_ms": 91.0, "p95_ms": 84.697}, "yolo11_640_fire_11class": {"avg_ms": 49.576, "count": 201, "max_ms": 67.922, "p95_ms": 64.226}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.413, "count": 201, "max_ms": 80.967, "p95_ms": 79.966}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.71, "count": 201, "max_ms": 162.782, "p95_ms": 156.319}, "read_interval": {"avg_ms": 148.957, "count": 202, "max_ms": 175.214, "p95_ms": 169.527}, "triton_wait": {"avg_ms": 65.724, "count": 201, "max_ms": 91.013, "p95_ms": 84.709}, "visualize": {"avg_ms": 4.309, "count": 201, "max_ms": 5.446, "p95_ms": 4.935}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11560}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10726.801, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:57:12+09:00", "last_error_summary": "[h264 @ 0x5627a0594f80] error while decoding MB 50 26, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 437, "h264_decode_error_count": 1337, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:57:12+09:00", "last_error_summary": "[h264 @ 0x5627a0594f80] error while decoding MB 50 26, bytestream -7", "last_exit_at": "2026-07-01T07:57:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.826, "last_started_at": "2026-07-01T07:54:28+09:00", "pid": 87931, "process": {"exists": false, "pid": 87931}, "restart_count": 478, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 479}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.843, "sample_count": 1985, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1387205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:57:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T07:57:12+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-01T07:57:12+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T07:57:12.634873+09:00", "mtime_epoch": 1782860232.6348727, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180865}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:57:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5027.133, "name": "celery", "pid": 267, "rss_mb": 5027.133, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.749, "pushed": 6.749, "read": 6.715}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.84, "count": 202, "max_ms": 43.121, "p95_ms": 36.952}, "gstreamer_push": {"avg_ms": 1.108, "count": 202, "max_ms": 1.492, "p95_ms": 1.159}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.534, "count": 202, "max_ms": 43.234, "p95_ms": 34.854}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.491, "count": 202, "max_ms": 91.0, "p95_ms": 84.697}, "yolo11_640_fire_11class": {"avg_ms": 49.573, "count": 202, "max_ms": 67.922, "p95_ms": 64.226}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.389, "count": 202, "max_ms": 80.967, "p95_ms": 79.966}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.675, "count": 202, "max_ms": 162.782, "p95_ms": 156.319}, "read_interval": {"avg_ms": 148.981, "count": 201, "max_ms": 175.214, "p95_ms": 169.527}, "triton_wait": {"avg_ms": 65.698, "count": 202, "max_ms": 91.013, "p95_ms": 84.709}, "visualize": {"avg_ms": 4.308, "count": 202, "max_ms": 5.446, "p95_ms": 4.935}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11560}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10726.828, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 437, "h264_decode_error_count": 1337, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:57:12+09:00", "last_error_summary": "[h264 @ 0x5627a0594f80] error while decoding MB 50 26, bytestream -7", "last_exit_at": "2026-07-01T07:57:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.831, "last_started_at": "2026-07-01T07:54:28+09:00", "pid": 87931, "process": {"exists": false, "pid": 87931}, "restart_count": 478, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 479}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.843, "sample_count": 1985, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1387205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:57:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T07:57:12+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-01T07:57:12+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T07:57:12.634873+09:00", "mtime_epoch": 1782860232.6348727, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180865}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:57:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5027.133, "name": "celery", "pid": 267, "rss_mb": 5027.133, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.748, "pushed": 6.748, "read": 6.714}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.84, "count": 202, "max_ms": 43.121, "p95_ms": 36.952}, "gstreamer_push": {"avg_ms": 1.108, "count": 202, "max_ms": 1.492, "p95_ms": 1.159}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.534, "count": 202, "max_ms": 43.234, "p95_ms": 34.854}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.491, "count": 202, "max_ms": 91.0, "p95_ms": 84.697}, "yolo11_640_fire_11class": {"avg_ms": 49.573, "count": 202, "max_ms": 67.922, "p95_ms": 64.226}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.389, "count": 202, "max_ms": 80.967, "p95_ms": 79.966}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.675, "count": 202, "max_ms": 162.782, "p95_ms": 156.319}, "read_interval": {"avg_ms": 148.981, "count": 201, "max_ms": 175.214, "p95_ms": 169.527}, "triton_wait": {"avg_ms": 65.698, "count": 202, "max_ms": 91.013, "p95_ms": 84.709}, "visualize": {"avg_ms": 4.308, "count": 202, "max_ms": 5.446, "p95_ms": 4.935}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11560}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10834.738, "limit_mb": 8796093022207.996}, "detail": {"pid": 88038, "restart_count": 479, "start_count": 480}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 437, "h264_decode_error_count": 1337, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:57:12+09:00", "last_error_summary": "[h264 @ 0x5627a0594f80] error while decoding MB 50 26, bytestream -7", "last_exit_at": "2026-07-01T07:57:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T07:57:15+09:00", "pid": 88038, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.523, "name": "ffmpeg", "pid": 88038, "rss_mb": 138.523, "state": "S (sleeping)", "threads": 106}, "restart_count": 479, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 480}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.821, "sample_count": 1952, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1387205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:57:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.884, "last_frame_pushed_at": "2026-07-01T07:57:12+09:00", "last_frame_read_age_sec": 5.016, "last_frame_read_at": "2026-07-01T07:57:12+09:00", "mjpeg": {"age_sec": 4.868, "exists": true, "mtime": "2026-07-01T07:57:12.774875+09:00", "mtime_epoch": 1782860232.7748754, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180516}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:57:17+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5027.133, "name": "celery", "pid": 267, "rss_mb": 5027.133, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 170, "pushed_frames": 170, "read_frames": 169, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.675, "pushed": 5.675, "read": 5.642}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.917, "count": 170, "max_ms": 43.121, "p95_ms": 37.225}, "gstreamer_push": {"avg_ms": 1.108, "count": 170, "max_ms": 1.492, "p95_ms": 1.163}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.761, "count": 170, "max_ms": 43.234, "p95_ms": 35.247}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.95, "count": 170, "max_ms": 91.0, "p95_ms": 84.851}, "yolo11_640_fire_11class": {"avg_ms": 49.305, "count": 170, "max_ms": 67.922, "p95_ms": 64.226}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.779, "count": 170, "max_ms": 80.967, "p95_ms": 79.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.149, "count": 170, "max_ms": 162.782, "p95_ms": 156.835}, "read_interval": {"avg_ms": 148.464, "count": 169, "max_ms": 175.214, "p95_ms": 169.798}, "triton_wait": {"avg_ms": 65.158, "count": 170, "max_ms": 91.013, "p95_ms": 84.864}, "visualize": {"avg_ms": 4.313, "count": 170, "max_ms": 5.446, "p95_ms": 4.997}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10729.332, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5606b85ecdc0] error while decoding MB 56 10, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 438, "h264_decode_error_count": 1338, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:59:59+09:00", "last_error_summary": "[h264 @ 0x5606b85ecdc0] error while decoding MB 56 10, bytestream -5", "last_exit_at": "2026-07-01T07:57:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.002, "last_started_at": "2026-07-01T07:57:15+09:00", "pid": 88038, "process": {"exists": false, "pid": 88038}, "restart_count": 479, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 480}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.958, "sample_count": 1982, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1388291, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:59:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T07:59:59+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T07:59:59+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T07:59:59.601934+09:00", "mtime_epoch": 1782860399.6019337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:59:59+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5027.648, "name": "celery", "pid": 267, "rss_mb": 5027.648, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.569, "pushed": 6.569, "read": 6.569}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.659, "count": 196, "max_ms": 37.472, "p95_ms": 36.055}, "gstreamer_push": {"avg_ms": 1.067, "count": 197, "max_ms": 1.271, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.913, "count": 196, "max_ms": 44.223, "p95_ms": 36.321}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.528, "count": 196, "max_ms": 86.594, "p95_ms": 82.532}, "yolo11_640_fire_11class": {"avg_ms": 49.105, "count": 196, "max_ms": 65.224, "p95_ms": 63.067}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.991, "count": 196, "max_ms": 80.109, "p95_ms": 77.878}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.847, "count": 197, "max_ms": 179.61, "p95_ms": 173.728}, "read_interval": {"avg_ms": 152.675, "count": 197, "max_ms": 191.916, "p95_ms": 184.823}, "triton_wait": {"avg_ms": 63.875, "count": 196, "max_ms": 86.605, "p95_ms": 82.544}, "visualize": {"avg_ms": 4.509, "count": 196, "max_ms": 5.646, "p95_ms": 5.465}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10728.688, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T07:59:59+09:00", "last_error_summary": "[h264 @ 0x5606b85ecdc0] error while decoding MB 56 10, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 438, "h264_decode_error_count": 1338, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:59:59+09:00", "last_error_summary": "[h264 @ 0x5606b85ecdc0] error while decoding MB 56 10, bytestream -5", "last_exit_at": "2026-07-01T07:59:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.096, "last_started_at": "2026-07-01T07:57:15+09:00", "pid": 88038, "process": {"exists": false, "pid": 88038}, "restart_count": 479, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 480}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.958, "sample_count": 1982, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1388292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:59:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T07:59:59+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-01T07:59:59+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-01T07:59:59.601934+09:00", "mtime_epoch": 1782860399.6019337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:59:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5027.648, "name": "celery", "pid": 267, "rss_mb": 5027.648, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 196, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.578, "pushed": 6.578, "read": 6.544}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.669, "count": 197, "max_ms": 37.472, "p95_ms": 36.055}, "gstreamer_push": {"avg_ms": 1.067, "count": 197, "max_ms": 1.271, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.896, "count": 197, "max_ms": 44.223, "p95_ms": 36.321}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.532, "count": 197, "max_ms": 86.594, "p95_ms": 82.532}, "yolo11_640_fire_11class": {"avg_ms": 49.082, "count": 197, "max_ms": 65.224, "p95_ms": 63.067}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.977, "count": 197, "max_ms": 80.109, "p95_ms": 77.878}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.853, "count": 197, "max_ms": 179.61, "p95_ms": 173.728}, "read_interval": {"avg_ms": 152.689, "count": 196, "max_ms": 191.916, "p95_ms": 184.823}, "triton_wait": {"avg_ms": 63.877, "count": 197, "max_ms": 86.605, "p95_ms": 82.544}, "visualize": {"avg_ms": 4.51, "count": 197, "max_ms": 5.646, "p95_ms": 5.465}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10728.68, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 438, "h264_decode_error_count": 1338, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:59:59+09:00", "last_error_summary": "[h264 @ 0x5606b85ecdc0] error while decoding MB 56 10, bytestream -5", "last_exit_at": "2026-07-01T07:59:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.102, "last_started_at": "2026-07-01T07:57:15+09:00", "pid": 88038, "process": {"exists": false, "pid": 88038}, "restart_count": 479, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 480}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.958, "sample_count": 1982, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1388292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:59:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T07:59:59+09:00", "last_frame_read_age_sec": 0.15, "last_frame_read_at": "2026-07-01T07:59:59+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-01T07:59:59.601934+09:00", "mtime_epoch": 1782860399.6019337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T07:59:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5027.648, "name": "celery", "pid": 267, "rss_mb": 5027.648, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 196, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.577, "pushed": 6.577, "read": 6.543}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.669, "count": 197, "max_ms": 37.472, "p95_ms": 36.055}, "gstreamer_push": {"avg_ms": 1.067, "count": 197, "max_ms": 1.271, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.896, "count": 197, "max_ms": 44.223, "p95_ms": 36.321}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.532, "count": 197, "max_ms": 86.594, "p95_ms": 82.532}, "yolo11_640_fire_11class": {"avg_ms": 49.082, "count": 197, "max_ms": 65.224, "p95_ms": 63.067}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.977, "count": 197, "max_ms": 80.109, "p95_ms": 77.878}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.853, "count": 197, "max_ms": 179.61, "p95_ms": 173.728}, "read_interval": {"avg_ms": 152.689, "count": 196, "max_ms": 191.916, "p95_ms": 184.823}, "triton_wait": {"avg_ms": 63.877, "count": 197, "max_ms": 86.605, "p95_ms": 82.544}, "visualize": {"avg_ms": 4.51, "count": 197, "max_ms": 5.646, "p95_ms": 5.465}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10836.66, "limit_mb": 8796093022207.996}, "detail": {"pid": 88145, "restart_count": 480, "start_count": 481}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 438, "h264_decode_error_count": 1338, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T07:59:59+09:00", "last_error_summary": "[h264 @ 0x5606b85ecdc0] error while decoding MB 56 10, bytestream -5", "last_exit_at": "2026-07-01T08:00:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:00:02+09:00", "pid": 88145, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.809, "name": "ffmpeg", "pid": 88145, "rss_mb": 139.809, "state": "S (sleeping)", "threads": 106}, "restart_count": 480, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 481}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.025, "sample_count": 1948, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1388292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T07:59:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.974, "last_frame_pushed_at": "2026-07-01T07:59:59+09:00", "last_frame_read_age_sec": 5.114, "last_frame_read_at": "2026-07-01T07:59:59+09:00", "mjpeg": {"age_sec": 4.958, "exists": true, "mtime": "2026-07-01T07:59:59.749937+09:00", "mtime_epoch": 1782860399.7499366, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177807}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:00:04+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5027.648, "name": "celery", "pid": 267, "rss_mb": 5027.648, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 161, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.424, "pushed": 5.424, "read": 5.39}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.794, "count": 162, "max_ms": 37.472, "p95_ms": 36.114}, "gstreamer_push": {"avg_ms": 1.06, "count": 162, "max_ms": 1.271, "p95_ms": 1.153}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.151, "count": 162, "max_ms": 44.223, "p95_ms": 36.589}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.917, "count": 162, "max_ms": 86.594, "p95_ms": 82.558}, "yolo11_640_fire_11class": {"avg_ms": 49.435, "count": 162, "max_ms": 65.224, "p95_ms": 63.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.464, "count": 162, "max_ms": 80.109, "p95_ms": 77.856}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.882, "count": 162, "max_ms": 179.61, "p95_ms": 175.191}, "read_interval": {"avg_ms": 154.673, "count": 161, "max_ms": 191.916, "p95_ms": 186.179}, "triton_wait": {"avg_ms": 64.283, "count": 162, "max_ms": 86.605, "p95_ms": 82.571}, "visualize": {"avg_ms": 4.582, "count": 162, "max_ms": 5.646, "p95_ms": 5.506}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11578}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10730.785, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ca8aecc380] error while decoding MB 64 34, bytestream -36", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 439, "h264_decode_error_count": 1339, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:02:45+09:00", "last_error_summary": "[h264 @ 0x55ca8aecc380] error while decoding MB 64 34, bytestream -36", "last_exit_at": "2026-07-01T08:00:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.709, "last_started_at": "2026-07-01T08:00:02+09:00", "pid": 88145, "process": {"exists": false, "pid": 88145}, "restart_count": 480, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 481}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.763, "sample_count": 1992, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1389384, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:02:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T08:02:45+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T08:02:45+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T08:02:45.372953+09:00", "mtime_epoch": 1782860565.3729532, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177983}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:02:45+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5028.164, "name": "celery", "pid": 267, "rss_mb": 5028.164, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.756, "pushed": 6.756, "read": 6.756}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.41, "count": 202, "max_ms": 38.842, "p95_ms": 35.786}, "gstreamer_push": {"avg_ms": 1.093, "count": 202, "max_ms": 1.193, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.32, "count": 202, "max_ms": 42.597, "p95_ms": 33.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.921, "count": 202, "max_ms": 89.619, "p95_ms": 84.98}, "yolo11_640_fire_11class": {"avg_ms": 48.818, "count": 202, "max_ms": 67.581, "p95_ms": 62.764}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.914, "count": 202, "max_ms": 79.495, "p95_ms": 72.693}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.242, "count": 202, "max_ms": 176.777, "p95_ms": 157.265}, "read_interval": {"avg_ms": 148.371, "count": 202, "max_ms": 187.094, "p95_ms": 168.827}, "triton_wait": {"avg_ms": 65.037, "count": 202, "max_ms": 89.631, "p95_ms": 84.995}, "visualize": {"avg_ms": 4.274, "count": 202, "max_ms": 5.61, "p95_ms": 4.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11578}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10730.195, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:02:45+09:00", "last_error_summary": "[h264 @ 0x55ca8aecc380] error while decoding MB 64 34, bytestream -36", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 439, "h264_decode_error_count": 1339, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:02:45+09:00", "last_error_summary": "[h264 @ 0x55ca8aecc380] error while decoding MB 64 34, bytestream -36", "last_exit_at": "2026-07-01T08:02:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.805, "last_started_at": "2026-07-01T08:00:02+09:00", "pid": 88145, "process": {"exists": false, "pid": 88145}, "restart_count": 480, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 481}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.763, "sample_count": 1992, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1389385, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:02:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T08:02:45+09:00", "last_frame_read_age_sec": 0.149, "last_frame_read_at": "2026-07-01T08:02:45+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-01T08:02:45.372953+09:00", "mtime_epoch": 1782860565.3729532, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177983}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:02:45+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5028.164, "name": "celery", "pid": 267, "rss_mb": 5028.164, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.767, "pushed": 6.767, "read": 6.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.427, "count": 202, "max_ms": 38.842, "p95_ms": 35.831}, "gstreamer_push": {"avg_ms": 1.093, "count": 203, "max_ms": 1.193, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.327, "count": 202, "max_ms": 42.597, "p95_ms": 33.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.982, "count": 202, "max_ms": 89.619, "p95_ms": 84.98}, "yolo11_640_fire_11class": {"avg_ms": 48.866, "count": 202, "max_ms": 67.581, "p95_ms": 62.764}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.957, "count": 202, "max_ms": 79.495, "p95_ms": 72.693}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.264, "count": 203, "max_ms": 176.777, "p95_ms": 157.265}, "read_interval": {"avg_ms": 148.371, "count": 202, "max_ms": 187.094, "p95_ms": 168.827}, "triton_wait": {"avg_ms": 65.098, "count": 202, "max_ms": 89.631, "p95_ms": 84.995}, "visualize": {"avg_ms": 4.274, "count": 202, "max_ms": 5.61, "p95_ms": 4.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11578}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10730.188, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 439, "h264_decode_error_count": 1339, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:02:45+09:00", "last_error_summary": "[h264 @ 0x55ca8aecc380] error while decoding MB 64 34, bytestream -36", "last_exit_at": "2026-07-01T08:02:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.81, "last_started_at": "2026-07-01T08:00:02+09:00", "pid": 88145, "process": {"exists": false, "pid": 88145}, "restart_count": 480, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 481}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.763, "sample_count": 1992, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1389385, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:02:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T08:02:45+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-01T08:02:45+09:00", "mjpeg": {"age_sec": 0.144, "exists": true, "mtime": "2026-07-01T08:02:45.372953+09:00", "mtime_epoch": 1782860565.3729532, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177983}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:02:45+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5028.164, "name": "celery", "pid": 267, "rss_mb": 5028.164, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 203, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.733, "pushed": 6.767, "read": 6.733}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.427, "count": 202, "max_ms": 38.842, "p95_ms": 35.831}, "gstreamer_push": {"avg_ms": 1.093, "count": 203, "max_ms": 1.193, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.327, "count": 202, "max_ms": 42.597, "p95_ms": 33.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.982, "count": 202, "max_ms": 89.619, "p95_ms": 84.98}, "yolo11_640_fire_11class": {"avg_ms": 48.866, "count": 202, "max_ms": 67.581, "p95_ms": 62.764}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.957, "count": 202, "max_ms": 79.495, "p95_ms": 72.693}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.327, "count": 202, "max_ms": 176.777, "p95_ms": 157.265}, "read_interval": {"avg_ms": 148.371, "count": 202, "max_ms": 187.094, "p95_ms": 168.827}, "triton_wait": {"avg_ms": 65.098, "count": 202, "max_ms": 89.631, "p95_ms": 84.995}, "visualize": {"avg_ms": 4.274, "count": 202, "max_ms": 5.61, "p95_ms": 4.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11578}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10840.793, "limit_mb": 8796093022207.996}, "detail": {"pid": 88252, "restart_count": 481, "start_count": 482}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 439, "h264_decode_error_count": 1339, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:02:45+09:00", "last_error_summary": "[h264 @ 0x55ca8aecc380] error while decoding MB 64 34, bytestream -36", "last_exit_at": "2026-07-01T08:02:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:02:48+09:00", "pid": 88252, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.273, "name": "ffmpeg", "pid": 88252, "rss_mb": 141.273, "state": "S (sleeping)", "threads": 106}, "restart_count": 481, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 482}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.888, "sample_count": 1959, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1389385, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:02:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.936, "last_frame_pushed_at": "2026-07-01T08:02:45+09:00", "last_frame_read_age_sec": 5.078, "last_frame_read_at": "2026-07-01T08:02:45+09:00", "mjpeg": {"age_sec": 4.92, "exists": true, "mtime": "2026-07-01T08:02:45.520956+09:00", "mtime_epoch": 1782860565.520956, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178022}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:02:50+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5028.164, "name": "celery", "pid": 267, "rss_mb": 5028.164, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 169, "pushed_frames": 169, "read_frames": 168, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.654, "pushed": 5.654, "read": 5.621}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.453, "count": 169, "max_ms": 38.061, "p95_ms": 36.012}, "gstreamer_push": {"avg_ms": 1.093, "count": 169, "max_ms": 1.183, "p95_ms": 1.153}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.257, "count": 169, "max_ms": 35.833, "p95_ms": 33.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.875, "count": 169, "max_ms": 89.619, "p95_ms": 84.821}, "yolo11_640_fire_11class": {"avg_ms": 49.116, "count": 169, "max_ms": 67.581, "p95_ms": 62.508}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.535, "count": 169, "max_ms": 78.94, "p95_ms": 72.33}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.332, "count": 169, "max_ms": 176.777, "p95_ms": 157.372}, "read_interval": {"avg_ms": 148.434, "count": 168, "max_ms": 187.094, "p95_ms": 168.827}, "triton_wait": {"avg_ms": 65.01, "count": 169, "max_ms": 89.631, "p95_ms": 84.832}, "visualize": {"avg_ms": 4.271, "count": 169, "max_ms": 5.61, "p95_ms": 4.527}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10932.887, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560263152780] error while decoding MB 67 21, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 440, "h264_decode_error_count": 1340, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:05:30+09:00", "last_error_summary": "[h264 @ 0x560263152780] error while decoding MB 67 21, bytestream -10", "last_exit_at": "2026-07-01T08:02:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.401, "last_started_at": "2026-07-01T08:02:48+09:00", "pid": 88252, "process": {"exists": false, "pid": 88252}, "restart_count": 481, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 482}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.951, "sample_count": 1990, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1390459, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:05:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T08:05:30+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T08:05:30+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T08:05:30.803937+09:00", "mtime_epoch": 1782860730.8039367, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:05:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5028.68, "name": "celery", "pid": 267, "rss_mb": 5028.68, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.812, "pushed": 6.812, "read": 6.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.388, "count": 204, "max_ms": 47.602, "p95_ms": 36.103}, "gstreamer_push": {"avg_ms": 1.086, "count": 204, "max_ms": 1.469, "p95_ms": 1.153}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.029, "count": 204, "max_ms": 49.249, "p95_ms": 33.555}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.44, "count": 204, "max_ms": 88.415, "p95_ms": 84.129}, "yolo11_640_fire_11class": {"avg_ms": 49.783, "count": 204, "max_ms": 65.41, "p95_ms": 63.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.889, "count": 204, "max_ms": 87.157, "p95_ms": 77.084}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.112, "count": 204, "max_ms": 159.977, "p95_ms": 155.782}, "read_interval": {"avg_ms": 147.269, "count": 204, "max_ms": 171.766, "p95_ms": 167.677}, "triton_wait": {"avg_ms": 64.73, "count": 204, "max_ms": 88.427, "p95_ms": 84.141}, "visualize": {"avg_ms": 4.29, "count": 204, "max_ms": 6.162, "p95_ms": 4.786}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10932.32, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:05:30+09:00", "last_error_summary": "[h264 @ 0x560263152780] error while decoding MB 67 21, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 440, "h264_decode_error_count": 1340, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:05:30+09:00", "last_error_summary": "[h264 @ 0x560263152780] error while decoding MB 67 21, bytestream -10", "last_exit_at": "2026-07-01T08:05:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.49, "last_started_at": "2026-07-01T08:02:48+09:00", "pid": 88252, "process": {"exists": false, "pid": 88252}, "restart_count": 481, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 482}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.951, "sample_count": 1990, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1390460, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:05:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T08:05:30+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-01T08:05:30+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-01T08:05:30.803937+09:00", "mtime_epoch": 1782860730.8039367, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:05:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5028.68, "name": "celery", "pid": 267, "rss_mb": 5028.68, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.824, "pushed": 6.824, "read": 6.791}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.405, "count": 204, "max_ms": 47.602, "p95_ms": 36.103}, "gstreamer_push": {"avg_ms": 1.086, "count": 204, "max_ms": 1.469, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.953, "count": 204, "max_ms": 49.249, "p95_ms": 33.407}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.358, "count": 204, "max_ms": 88.415, "p95_ms": 84.129}, "yolo11_640_fire_11class": {"avg_ms": 49.738, "count": 204, "max_ms": 65.41, "p95_ms": 63.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.796, "count": 204, "max_ms": 87.157, "p95_ms": 77.084}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.045, "count": 204, "max_ms": 159.977, "p95_ms": 155.782}, "read_interval": {"avg_ms": 147.21, "count": 203, "max_ms": 171.766, "p95_ms": 167.677}, "triton_wait": {"avg_ms": 64.648, "count": 204, "max_ms": 88.427, "p95_ms": 84.141}, "visualize": {"avg_ms": 4.291, "count": 204, "max_ms": 6.162, "p95_ms": 4.786}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10932.324, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 440, "h264_decode_error_count": 1340, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:05:30+09:00", "last_error_summary": "[h264 @ 0x560263152780] error while decoding MB 67 21, bytestream -10", "last_exit_at": "2026-07-01T08:05:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.494, "last_started_at": "2026-07-01T08:02:48+09:00", "pid": 88252, "process": {"exists": false, "pid": 88252}, "restart_count": 481, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 482}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.951, "sample_count": 1990, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1390460, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:05:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T08:05:30+09:00", "last_frame_read_age_sec": 0.146, "last_frame_read_at": "2026-07-01T08:05:30+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-01T08:05:30.803937+09:00", "mtime_epoch": 1782860730.8039367, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:05:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5028.68, "name": "celery", "pid": 267, "rss_mb": 5028.68, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.823, "pushed": 6.823, "read": 6.79}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.405, "count": 204, "max_ms": 47.602, "p95_ms": 36.103}, "gstreamer_push": {"avg_ms": 1.086, "count": 204, "max_ms": 1.469, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.953, "count": 204, "max_ms": 49.249, "p95_ms": 33.407}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.358, "count": 204, "max_ms": 88.415, "p95_ms": 84.129}, "yolo11_640_fire_11class": {"avg_ms": 49.738, "count": 204, "max_ms": 65.41, "p95_ms": 63.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.796, "count": 204, "max_ms": 87.157, "p95_ms": 77.084}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.045, "count": 204, "max_ms": 159.977, "p95_ms": 155.782}, "read_interval": {"avg_ms": 147.21, "count": 203, "max_ms": 171.766, "p95_ms": 167.677}, "triton_wait": {"avg_ms": 64.648, "count": 204, "max_ms": 88.427, "p95_ms": 84.141}, "visualize": {"avg_ms": 4.291, "count": 204, "max_ms": 6.162, "p95_ms": 4.786}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10986.012, "limit_mb": 8796093022207.996}, "detail": {"pid": 88419, "restart_count": 482, "start_count": 483}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 440, "h264_decode_error_count": 1340, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:05:30+09:00", "last_error_summary": "[h264 @ 0x560263152780] error while decoding MB 67 21, bytestream -10", "last_exit_at": "2026-07-01T08:05:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:05:33+09:00", "pid": 88419, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.02, "name": "ffmpeg", "pid": 88419, "rss_mb": 139.02, "state": "S (sleeping)", "threads": 106}, "restart_count": 482, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 483}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.921, "sample_count": 1955, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1390460, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:05:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 5.018, "last_frame_pushed_at": "2026-07-01T08:05:30+09:00", "last_frame_read_age_sec": 5.153, "last_frame_read_at": "2026-07-01T08:05:30+09:00", "mjpeg": {"age_sec": 4.998, "exists": true, "mtime": "2026-07-01T08:05:30.943938+09:00", "mtime_epoch": 1782860730.9439385, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177741}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:05:35+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5028.68, "name": "celery", "pid": 267, "rss_mb": 5028.68, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 171, "pushed_frames": 171, "read_frames": 170, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.724, "pushed": 5.724, "read": 5.69}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.336, "count": 171, "max_ms": 47.602, "p95_ms": 35.95}, "gstreamer_push": {"avg_ms": 1.078, "count": 171, "max_ms": 1.231, "p95_ms": 1.147}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.983, "count": 171, "max_ms": 49.249, "p95_ms": 33.802}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.862, "count": 171, "max_ms": 88.415, "p95_ms": 83.297}, "yolo11_640_fire_11class": {"avg_ms": 49.32, "count": 171, "max_ms": 65.41, "p95_ms": 62.711}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.217, "count": 171, "max_ms": 87.157, "p95_ms": 77.346}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.291, "count": 171, "max_ms": 159.977, "p95_ms": 153.997}, "read_interval": {"avg_ms": 146.354, "count": 170, "max_ms": 171.766, "p95_ms": 167.309}, "triton_wait": {"avg_ms": 64.092, "count": 171, "max_ms": 88.427, "p95_ms": 83.31}, "visualize": {"avg_ms": 4.275, "count": 171, "max_ms": 6.162, "p95_ms": 4.603}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11596}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11100.824, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f30b2fe980] error while decoding MB 21 24, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 441, "h264_decode_error_count": 1341, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:08:19+09:00", "last_error_summary": "[h264 @ 0x55f30b2fe980] error while decoding MB 21 24, bytestream -5", "last_exit_at": "2026-07-01T08:05:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 165.114, "last_started_at": "2026-07-01T08:05:33+09:00", "pid": 88419, "process": {"exists": false, "pid": 88419}, "restart_count": 482, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 483}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.274, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1391576, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:08:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T08:08:18+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T08:08:19+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T08:08:19.010181+09:00", "mtime_epoch": 1782860899.0101814, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176776}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:08:19+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5029.195, "name": "celery", "pid": 267, "rss_mb": 5029.195, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 192, "pushed_frames": 192, "read_frames": 192, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.426, "pushed": 6.426, "read": 6.426}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.369, "count": 192, "max_ms": 37.041, "p95_ms": 35.336}, "gstreamer_push": {"avg_ms": 1.091, "count": 192, "max_ms": 1.532, "p95_ms": 1.165}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.282, "count": 192, "max_ms": 41.321, "p95_ms": 36.574}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.547, "count": 192, "max_ms": 93.366, "p95_ms": 84.403}, "yolo11_640_fire_11class": {"avg_ms": 48.735, "count": 192, "max_ms": 74.016, "p95_ms": 62.637}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.802, "count": 192, "max_ms": 88.935, "p95_ms": 76.528}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.166, "count": 192, "max_ms": 162.176, "p95_ms": 155.549}, "read_interval": {"avg_ms": 156.019, "count": 192, "max_ms": 2110.919, "p95_ms": 168.443}, "triton_wait": {"avg_ms": 63.685, "count": 192, "max_ms": 93.378, "p95_ms": 84.415}, "visualize": {"avg_ms": 4.25, "count": 192, "max_ms": 5.198, "p95_ms": 4.679}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11596}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11106.812, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:08:19+09:00", "last_error_summary": "[h264 @ 0x55f30b2fe980] error while decoding MB 21 24, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 441, "h264_decode_error_count": 1341, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:08:19+09:00", "last_error_summary": "[h264 @ 0x55f30b2fe980] error while decoding MB 21 24, bytestream -5", "last_exit_at": "2026-07-01T08:08:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 165.199, "last_started_at": "2026-07-01T08:05:33+09:00", "pid": 88419, "process": {"exists": false, "pid": 88419}, "restart_count": 482, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 483}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.274, "sample_count": 2010, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1391577, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:08:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:08:19+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-01T08:08:19+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T08:08:19.010181+09:00", "mtime_epoch": 1782860899.0101814, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176776}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:08:19+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5029.195, "name": "celery", "pid": 267, "rss_mb": 5029.195, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 193, "pushed_frames": 193, "read_frames": 192, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.441, "pushed": 6.441, "read": 6.407}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.37, "count": 192, "max_ms": 37.041, "p95_ms": 35.336}, "gstreamer_push": {"avg_ms": 1.09, "count": 193, "max_ms": 1.532, "p95_ms": 1.165}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.238, "count": 192, "max_ms": 41.321, "p95_ms": 36.113}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.563, "count": 192, "max_ms": 93.366, "p95_ms": 84.403}, "yolo11_640_fire_11class": {"avg_ms": 48.727, "count": 192, "max_ms": 74.016, "p95_ms": 62.637}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.8, "count": 192, "max_ms": 88.935, "p95_ms": 76.528}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.155, "count": 193, "max_ms": 162.176, "p95_ms": 155.549}, "read_interval": {"avg_ms": 156.019, "count": 192, "max_ms": 2110.919, "p95_ms": 168.443}, "triton_wait": {"avg_ms": 63.701, "count": 192, "max_ms": 93.378, "p95_ms": 84.415}, "visualize": {"avg_ms": 4.25, "count": 192, "max_ms": 5.198, "p95_ms": 4.679}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11596}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11106.805, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 441, "h264_decode_error_count": 1341, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:08:19+09:00", "last_error_summary": "[h264 @ 0x55f30b2fe980] error while decoding MB 21 24, bytestream -5", "last_exit_at": "2026-07-01T08:08:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 165.204, "last_started_at": "2026-07-01T08:05:33+09:00", "pid": 88419, "process": {"exists": false, "pid": 88419}, "restart_count": 482, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 483}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.274, "sample_count": 2010, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1391577, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:08:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:08:19+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-01T08:08:19+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T08:08:19.010181+09:00", "mtime_epoch": 1782860899.0101814, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176776}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:08:19+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5029.195, "name": "celery", "pid": 267, "rss_mb": 5029.195, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 193, "pushed_frames": 193, "read_frames": 192, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.44, "pushed": 6.44, "read": 6.406}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.37, "count": 192, "max_ms": 37.041, "p95_ms": 35.336}, "gstreamer_push": {"avg_ms": 1.09, "count": 193, "max_ms": 1.532, "p95_ms": 1.165}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.238, "count": 192, "max_ms": 41.321, "p95_ms": 36.113}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.563, "count": 192, "max_ms": 93.366, "p95_ms": 84.403}, "yolo11_640_fire_11class": {"avg_ms": 48.727, "count": 192, "max_ms": 74.016, "p95_ms": 62.637}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.8, "count": 192, "max_ms": 88.935, "p95_ms": 76.528}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.155, "count": 193, "max_ms": 162.176, "p95_ms": 155.549}, "read_interval": {"avg_ms": 156.019, "count": 192, "max_ms": 2110.919, "p95_ms": 168.443}, "triton_wait": {"avg_ms": 63.701, "count": 192, "max_ms": 93.378, "p95_ms": 84.415}, "visualize": {"avg_ms": 4.25, "count": 192, "max_ms": 5.198, "p95_ms": 4.679}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11596}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11294.906, "limit_mb": 8796093022207.996}, "detail": {"pid": 88646, "restart_count": 483, "start_count": 484}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 441, "h264_decode_error_count": 1341, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:08:19+09:00", "last_error_summary": "[h264 @ 0x55f30b2fe980] error while decoding MB 21 24, bytestream -5", "last_exit_at": "2026-07-01T08:08:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:08:21+09:00", "pid": 88646, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.051, "name": "ffmpeg", "pid": 88646, "rss_mb": 139.051, "state": "S (sleeping)", "threads": 106}, "restart_count": 483, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 484}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.761, "sample_count": 1980, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1391577, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:08:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.761, "last_frame_pushed_at": "2026-07-01T08:08:19+09:00", "last_frame_read_age_sec": 4.894, "last_frame_read_at": "2026-07-01T08:08:19+09:00", "mjpeg": {"age_sec": 4.746, "exists": true, "mtime": "2026-07-01T08:08:19.150183+09:00", "mtime_epoch": 1782860899.1501832, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177066}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:08:23+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5029.195, "name": "celery", "pid": 267, "rss_mb": 5029.195, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.383, "pushed": 5.383, "read": 5.35}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.308, "count": 161, "max_ms": 37.041, "p95_ms": 35.205}, "gstreamer_push": {"avg_ms": 1.09, "count": 161, "max_ms": 1.532, "p95_ms": 1.165}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.182, "count": 161, "max_ms": 41.321, "p95_ms": 35.756}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.858, "count": 161, "max_ms": 93.366, "p95_ms": 84.403}, "yolo11_640_fire_11class": {"avg_ms": 48.063, "count": 161, "max_ms": 74.016, "p95_ms": 62.208}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.026, "count": 161, "max_ms": 88.935, "p95_ms": 75.203}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.34, "count": 161, "max_ms": 162.176, "p95_ms": 155.549}, "read_interval": {"avg_ms": 157.148, "count": 160, "max_ms": 2110.919, "p95_ms": 168.443}, "triton_wait": {"avg_ms": 62.972, "count": 161, "max_ms": 93.378, "p95_ms": 84.415}, "visualize": {"avg_ms": 4.242, "count": 161, "max_ms": 5.032, "p95_ms": 4.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10914.508, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5585afaf1e40] error while decoding MB 79 34, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 442, "h264_decode_error_count": 1342, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:11:05+09:00", "last_error_summary": "[h264 @ 0x5585afaf1e40] error while decoding MB 79 34, bytestream -7", "last_exit_at": "2026-07-01T08:08:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.39, "last_started_at": "2026-07-01T08:08:21+09:00", "pid": 88646, "process": {"exists": false, "pid": 88646}, "restart_count": 483, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 484}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.659, "sample_count": 2015, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1392683, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:11:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T08:11:05+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T08:11:05+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T08:11:05.240573+09:00", "mtime_epoch": 1782861065.240573, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176588}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:11:05+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5029.711, "name": "celery", "pid": 267, "rss_mb": 5029.711, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 211, "pushed_frames": 211, "read_frames": 212, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.035, "pushed": 7.035, "read": 7.068}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.234, "count": 211, "max_ms": 37.997, "p95_ms": 34.902}, "gstreamer_push": {"avg_ms": 1.071, "count": 211, "max_ms": 1.17, "p95_ms": 1.141}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.468, "count": 211, "max_ms": 43.9, "p95_ms": 40.705}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.372, "count": 211, "max_ms": 88.711, "p95_ms": 75.003}, "yolo11_640_fire_11class": {"avg_ms": 47.172, "count": 211, "max_ms": 68.867, "p95_ms": 55.709}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.661, "count": 211, "max_ms": 81.569, "p95_ms": 70.264}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.882, "count": 211, "max_ms": 159.514, "p95_ms": 146.895}, "read_interval": {"avg_ms": 141.883, "count": 212, "max_ms": 171.313, "p95_ms": 159.239}, "triton_wait": {"avg_ms": 60.429, "count": 211, "max_ms": 88.723, "p95_ms": 75.117}, "visualize": {"avg_ms": 4.244, "count": 211, "max_ms": 5.57, "p95_ms": 4.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10913.867, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:11:05+09:00", "last_error_summary": "[h264 @ 0x5585afaf1e40] error while decoding MB 79 34, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 442, "h264_decode_error_count": 1342, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:11:05+09:00", "last_error_summary": "[h264 @ 0x5585afaf1e40] error while decoding MB 79 34, bytestream -7", "last_exit_at": "2026-07-01T08:11:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.493, "last_started_at": "2026-07-01T08:08:21+09:00", "pid": 88646, "process": {"exists": false, "pid": 88646}, "restart_count": 483, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 484}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.659, "sample_count": 2015, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1392684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:11:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:11:05+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-01T08:11:05+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-01T08:11:05.240573+09:00", "mtime_epoch": 1782861065.240573, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176588}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:11:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5029.711, "name": "celery", "pid": 267, "rss_mb": 5029.711, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.075, "pushed": 7.075, "read": 7.041}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.231, "count": 212, "max_ms": 37.997, "p95_ms": 34.902}, "gstreamer_push": {"avg_ms": 1.072, "count": 212, "max_ms": 1.217, "p95_ms": 1.146}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.477, "count": 212, "max_ms": 43.9, "p95_ms": 40.705}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.442, "count": 212, "max_ms": 88.711, "p95_ms": 75.105}, "yolo11_640_fire_11class": {"avg_ms": 47.224, "count": 212, "max_ms": 68.867, "p95_ms": 56.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.731, "count": 212, "max_ms": 81.569, "p95_ms": 70.264}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.956, "count": 212, "max_ms": 159.514, "p95_ms": 146.895}, "read_interval": {"avg_ms": 141.912, "count": 211, "max_ms": 171.313, "p95_ms": 159.239}, "triton_wait": {"avg_ms": 60.498, "count": 212, "max_ms": 88.723, "p95_ms": 75.125}, "visualize": {"avg_ms": 4.244, "count": 212, "max_ms": 5.57, "p95_ms": 4.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10913.859, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 442, "h264_decode_error_count": 1342, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:11:05+09:00", "last_error_summary": "[h264 @ 0x5585afaf1e40] error while decoding MB 79 34, bytestream -7", "last_exit_at": "2026-07-01T08:11:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.499, "last_started_at": "2026-07-01T08:08:21+09:00", "pid": 88646, "process": {"exists": false, "pid": 88646}, "restart_count": 483, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 484}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.659, "sample_count": 2015, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1392684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:11:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:11:05+09:00", "last_frame_read_age_sec": 0.158, "last_frame_read_at": "2026-07-01T08:11:05+09:00", "mjpeg": {"age_sec": 0.153, "exists": true, "mtime": "2026-07-01T08:11:05.240573+09:00", "mtime_epoch": 1782861065.240573, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176588}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:11:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5029.711, "name": "celery", "pid": 267, "rss_mb": 5029.711, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.074, "pushed": 7.074, "read": 7.04}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.208, "count": 211, "max_ms": 37.997, "p95_ms": 34.885}, "gstreamer_push": {"avg_ms": 1.072, "count": 212, "max_ms": 1.217, "p95_ms": 1.146}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.48, "count": 211, "max_ms": 43.9, "p95_ms": 40.705}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.465, "count": 211, "max_ms": 88.711, "p95_ms": 75.105}, "yolo11_640_fire_11class": {"avg_ms": 47.24, "count": 211, "max_ms": 68.867, "p95_ms": 56.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.756, "count": 211, "max_ms": 81.569, "p95_ms": 70.264}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.956, "count": 212, "max_ms": 159.514, "p95_ms": 146.895}, "read_interval": {"avg_ms": 141.912, "count": 211, "max_ms": 171.313, "p95_ms": 159.239}, "triton_wait": {"avg_ms": 60.521, "count": 211, "max_ms": 88.723, "p95_ms": 75.125}, "visualize": {"avg_ms": 4.244, "count": 211, "max_ms": 5.57, "p95_ms": 4.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11019.551, "limit_mb": 8796093022207.996}, "detail": {"pid": 88813, "restart_count": 484, "start_count": 485}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 442, "h264_decode_error_count": 1342, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:11:05+09:00", "last_error_summary": "[h264 @ 0x5585afaf1e40] error while decoding MB 79 34, bytestream -7", "last_exit_at": "2026-07-01T08:11:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:11:07+09:00", "pid": 88813, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.348, "name": "ffmpeg", "pid": 88813, "rss_mb": 136.348, "state": "S (sleeping)", "threads": 106}, "restart_count": 484, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 485}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.672, "sample_count": 1983, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1392684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:11:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.545, "last_frame_pushed_at": "2026-07-01T08:11:05+09:00", "last_frame_read_age_sec": 4.693, "last_frame_read_at": "2026-07-01T08:11:05+09:00", "mjpeg": {"age_sec": 4.528, "exists": true, "mtime": "2026-07-01T08:11:05.400575+09:00", "mtime_epoch": 1782861065.4005752, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176529}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:11:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5029.711, "name": "celery", "pid": 267, "rss_mb": 5029.711, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.013, "pushed": 6.013, "read": 5.979}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.849, "count": 180, "max_ms": 37.997, "p95_ms": 34.156}, "gstreamer_push": {"avg_ms": 1.074, "count": 180, "max_ms": 1.217, "p95_ms": 1.141}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.513, "count": 180, "max_ms": 43.9, "p95_ms": 40.705}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.58, "count": 180, "max_ms": 88.711, "p95_ms": 74.976}, "yolo11_640_fire_11class": {"avg_ms": 47.371, "count": 180, "max_ms": 68.867, "p95_ms": 55.709}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.736, "count": 180, "max_ms": 78.682, "p95_ms": 70.034}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.748, "count": 180, "max_ms": 159.514, "p95_ms": 145.886}, "read_interval": {"avg_ms": 141.733, "count": 179, "max_ms": 171.313, "p95_ms": 158.163}, "triton_wait": {"avg_ms": 60.628, "count": 180, "max_ms": 88.723, "p95_ms": 75.1}, "visualize": {"avg_ms": 4.24, "count": 180, "max_ms": 5.57, "p95_ms": 4.478}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 65, "tracking_summary": 13087}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10862.016, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b694b8ad00] error while decoding MB 19 62, bytestream -15 [h264 @ 0x55b6955c5740] error while decoding MB 27 52, bytestream -37", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 199, "h264_decode_error_count": 205, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:13:39+09:00", "last_error_summary": "[h264 @ 0x55b694b8ad00] error while decoding MB 19 62, bytestream -15 [h264 @ 0x55b6955c5740] error while decoding MB 27 52, bytestream -37", "last_exit_at": "2026-07-01T06:45:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 5310.647, "last_started_at": "2026-07-01T06:45:08+09:00", "pid": 84984, "process": {"exists": false, "pid": 84984}, "restart_count": 198, "restart_count_per_hour": 0, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 199}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 196.85, "sample_count": 2077, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1570537, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:13:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-01T08:13:39+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-01T08:13:39+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T08:13:39.438903+09:00", "mtime_epoch": 1782861219.438903, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 212945}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:13:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1944.27, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.709, "pushed": 6.709, "read": 6.709}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.983, "count": 200, "max_ms": 32.243, "p95_ms": 26.418}, "gstreamer_push": {"avg_ms": 1.13, "count": 201, "max_ms": 2.422, "p95_ms": 1.845}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.21, "count": 200, "max_ms": 64.209, "p95_ms": 58.037}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.231, "count": 200, "max_ms": 84.94, "p95_ms": 82.22}, "yolo11_640_fire_11class": {"avg_ms": 50.341, "count": 200, "max_ms": 67.529, "p95_ms": 60.383}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.493, "count": 200, "max_ms": 84.782, "p95_ms": 78.445}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.22, "count": 201, "max_ms": 788.105, "p95_ms": 198.756}, "read_interval": {"avg_ms": 149.58, "count": 201, "max_ms": 799.145, "p95_ms": 208.614}, "triton_wait": {"avg_ms": 61.636, "count": 200, "max_ms": 84.954, "p95_ms": 82.232}, "visualize": {"avg_ms": 6.359, "count": 200, "max_ms": 15.053, "p95_ms": 12.313}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 65, "tracking_summary": 13087}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10861.957, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:13:39+09:00", "last_error_summary": "[h264 @ 0x55b694b8ad00] error while decoding MB 19 62, bytestream -15 [h264 @ 0x55b6955c5740] error while decoding MB 27 52, bytestream -37", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 199, "h264_decode_error_count": 205, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:13:39+09:00", "last_error_summary": "[h264 @ 0x55b694b8ad00] error while decoding MB 19 62, bytestream -15 [h264 @ 0x55b6955c5740] error while decoding MB 27 52, bytestream -37", "last_exit_at": "2026-07-01T08:13:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 5310.672, "last_started_at": "2026-07-01T06:45:08+09:00", "pid": 84984, "process": {"exists": false, "pid": 84984}, "restart_count": 198, "restart_count_per_hour": 0, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 199}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 196.85, "sample_count": 2078, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1570538, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:13:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T08:13:39+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T08:13:39+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-01T08:13:39.438903+09:00", "mtime_epoch": 1782861219.438903, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 212945}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:13:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1944.27, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.737, "pushed": 6.737, "read": 6.703}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.994, "count": 201, "max_ms": 32.243, "p95_ms": 26.418}, "gstreamer_push": {"avg_ms": 1.13, "count": 202, "max_ms": 2.422, "p95_ms": 1.845}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.213, "count": 201, "max_ms": 64.209, "p95_ms": 58.037}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.181, "count": 201, "max_ms": 84.94, "p95_ms": 82.22}, "yolo11_640_fire_11class": {"avg_ms": 50.326, "count": 201, "max_ms": 67.529, "p95_ms": 60.383}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.446, "count": 201, "max_ms": 84.782, "p95_ms": 78.445}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.959, "count": 202, "max_ms": 788.105, "p95_ms": 198.756}, "read_interval": {"avg_ms": 149.58, "count": 201, "max_ms": 799.145, "p95_ms": 208.614}, "triton_wait": {"avg_ms": 61.584, "count": 201, "max_ms": 84.954, "p95_ms": 82.232}, "visualize": {"avg_ms": 6.353, "count": 201, "max_ms": 15.053, "p95_ms": 12.313}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 65, "tracking_summary": 13087}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10862.074, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 199, "h264_decode_error_count": 205, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:13:39+09:00", "last_error_summary": "[h264 @ 0x55b694b8ad00] error while decoding MB 19 62, bytestream -15 [h264 @ 0x55b6955c5740] error while decoding MB 27 52, bytestream -37", "last_exit_at": "2026-07-01T08:13:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 5310.68, "last_started_at": "2026-07-01T06:45:08+09:00", "pid": 84984, "process": {"exists": false, "pid": 84984}, "restart_count": 198, "restart_count_per_hour": 0, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 199}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 196.85, "sample_count": 2078, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1570538, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:13:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T08:13:39+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T08:13:39+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-01T08:13:39.438903+09:00", "mtime_epoch": 1782861219.438903, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 212945}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:13:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1944.27, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.735, "pushed": 6.735, "read": 6.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.994, "count": 201, "max_ms": 32.243, "p95_ms": 26.418}, "gstreamer_push": {"avg_ms": 1.13, "count": 202, "max_ms": 2.422, "p95_ms": 1.845}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.213, "count": 201, "max_ms": 64.209, "p95_ms": 58.037}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.181, "count": 201, "max_ms": 84.94, "p95_ms": 82.22}, "yolo11_640_fire_11class": {"avg_ms": 50.326, "count": 201, "max_ms": 67.529, "p95_ms": 60.383}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.446, "count": 201, "max_ms": 84.782, "p95_ms": 78.445}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.959, "count": 202, "max_ms": 788.105, "p95_ms": 198.756}, "read_interval": {"avg_ms": 149.58, "count": 201, "max_ms": 799.145, "p95_ms": 208.614}, "triton_wait": {"avg_ms": 61.584, "count": 201, "max_ms": 84.954, "p95_ms": 82.232}, "visualize": {"avg_ms": 6.353, "count": 201, "max_ms": 15.053, "p95_ms": 12.313}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 65, "tracking_summary": 13087}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10883.57, "limit_mb": 8796093022207.996}, "detail": {"pid": 88980, "restart_count": 199, "start_count": 200}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 199, "h264_decode_error_count": 205, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:13:39+09:00", "last_error_summary": "[h264 @ 0x55b694b8ad00] error while decoding MB 19 62, bytestream -15 [h264 @ 0x55b6955c5740] error while decoding MB 27 52, bytestream -37", "last_exit_at": "2026-07-01T08:13:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:13:41+09:00", "pid": 88980, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.008, "name": "ffmpeg", "pid": 88980, "rss_mb": 58.008, "state": "S (sleeping)", "threads": 1}, "restart_count": 199, "restart_count_per_hour": 1, "rtsp_timeout_count": 324, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 200}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 197.028, "sample_count": 2050, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1570538, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:13:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.357, "last_frame_pushed_at": "2026-07-01T08:13:39+09:00", "last_frame_read_age_sec": 4.444, "last_frame_read_at": "2026-07-01T08:13:39+09:00", "mjpeg": {"age_sec": 4.337, "exists": true, "mtime": "2026-07-01T08:13:39.538905+09:00", "mtime_epoch": 1782861219.5389047, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 216779}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:13:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1944.27, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.984, "pushed": 5.984, "read": 5.951}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.595, "count": 179, "max_ms": 32.243, "p95_ms": 25.965}, "gstreamer_push": {"avg_ms": 1.163, "count": 179, "max_ms": 2.422, "p95_ms": 1.853}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.753, "count": 179, "max_ms": 64.209, "p95_ms": 58.145}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.635, "count": 179, "max_ms": 84.94, "p95_ms": 81.367}, "yolo11_640_fire_11class": {"avg_ms": 50.24, "count": 179, "max_ms": 67.529, "p95_ms": 60.703}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.611, "count": 179, "max_ms": 84.782, "p95_ms": 76.667}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.27, "count": 179, "max_ms": 788.105, "p95_ms": 196.249}, "read_interval": {"avg_ms": 144.184, "count": 178, "max_ms": 799.145, "p95_ms": 207.304}, "triton_wait": {"avg_ms": 60.98, "count": 179, "max_ms": 84.954, "p95_ms": 81.379}, "visualize": {"avg_ms": 5.832, "count": 179, "max_ms": 15.053, "p95_ms": 12.639}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10891.062, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e49b0ecf80] error while decoding MB 56 19, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 443, "h264_decode_error_count": 1343, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:14:07+09:00", "last_error_summary": "[h264 @ 0x55e49b0ecf80] error while decoding MB 56 19, bytestream -12", "last_exit_at": "2026-07-01T08:11:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 179.271, "last_started_at": "2026-07-01T08:11:07+09:00", "pid": 88813, "process": {"exists": false, "pid": 88813}, "restart_count": 484, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 485}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.162, "sample_count": 2046, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1393924, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:14:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T08:14:07+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T08:14:07+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T08:14:07.159331+09:00", "mtime_epoch": 1782861247.159331, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174610}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:14:07+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5030.227, "name": "celery", "pid": 267, "rss_mb": 5030.227, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.898, "pushed": 6.898, "read": 6.898}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.2, "count": 206, "max_ms": 38.823, "p95_ms": 36.327}, "gstreamer_push": {"avg_ms": 1.106, "count": 206, "max_ms": 1.627, "p95_ms": 1.253}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.458, "count": 206, "max_ms": 42.562, "p95_ms": 35.143}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.666, "count": 206, "max_ms": 88.142, "p95_ms": 83.669}, "yolo11_640_fire_11class": {"avg_ms": 50.227, "count": 206, "max_ms": 68.321, "p95_ms": 64.296}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.143, "count": 206, "max_ms": 82.958, "p95_ms": 78.513}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.06, "count": 206, "max_ms": 160.499, "p95_ms": 155.493}, "read_interval": {"avg_ms": 145.466, "count": 206, "max_ms": 175.422, "p95_ms": 168.587}, "triton_wait": {"avg_ms": 63.822, "count": 206, "max_ms": 88.153, "p95_ms": 83.681}, "visualize": {"avg_ms": 4.359, "count": 206, "max_ms": 6.782, "p95_ms": 5.167}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.865, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10890.234, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:14:07+09:00", "last_error_summary": "[h264 @ 0x55e49b0ecf80] error while decoding MB 56 19, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 443, "h264_decode_error_count": 1343, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:14:07+09:00", "last_error_summary": "[h264 @ 0x55e49b0ecf80] error while decoding MB 56 19, bytestream -12", "last_exit_at": "2026-07-01T08:14:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 179.375, "last_started_at": "2026-07-01T08:11:07+09:00", "pid": 88813, "process": {"exists": false, "pid": 88813}, "restart_count": 484, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 485}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.162, "sample_count": 2047, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1393925, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:14:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:14:07+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-01T08:14:07+09:00", "mjpeg": {"age_sec": 0.144, "exists": true, "mtime": "2026-07-01T08:14:07.159331+09:00", "mtime_epoch": 1782861247.159331, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174610}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:14:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5030.227, "name": "celery", "pid": 267, "rss_mb": 5030.227, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.907, "pushed": 6.907, "read": 6.874}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.185, "count": 206, "max_ms": 38.823, "p95_ms": 36.327}, "gstreamer_push": {"avg_ms": 1.106, "count": 207, "max_ms": 1.627, "p95_ms": 1.253}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.506, "count": 206, "max_ms": 42.562, "p95_ms": 35.275}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.642, "count": 206, "max_ms": 88.142, "p95_ms": 83.669}, "yolo11_640_fire_11class": {"avg_ms": 50.218, "count": 206, "max_ms": 68.321, "p95_ms": 64.296}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.145, "count": 206, "max_ms": 82.958, "p95_ms": 78.513}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.138, "count": 207, "max_ms": 160.499, "p95_ms": 155.493}, "read_interval": {"avg_ms": 145.466, "count": 206, "max_ms": 175.422, "p95_ms": 168.587}, "triton_wait": {"avg_ms": 63.798, "count": 206, "max_ms": 88.153, "p95_ms": 83.681}, "visualize": {"avg_ms": 4.359, "count": 206, "max_ms": 6.782, "p95_ms": 5.167}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10890.227, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 443, "h264_decode_error_count": 1343, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:14:07+09:00", "last_error_summary": "[h264 @ 0x55e49b0ecf80] error while decoding MB 56 19, bytestream -12", "last_exit_at": "2026-07-01T08:14:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 179.381, "last_started_at": "2026-07-01T08:11:07+09:00", "pid": 88813, "process": {"exists": false, "pid": 88813}, "restart_count": 484, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 485}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.162, "sample_count": 2046, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1393925, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:14:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:14:07+09:00", "last_frame_read_age_sec": 0.161, "last_frame_read_at": "2026-07-01T08:14:07+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-01T08:14:07.159331+09:00", "mtime_epoch": 1782861247.159331, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174610}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:14:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5030.227, "name": "celery", "pid": 267, "rss_mb": 5030.227, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.906, "pushed": 6.906, "read": 6.873}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.185, "count": 206, "max_ms": 38.823, "p95_ms": 36.327}, "gstreamer_push": {"avg_ms": 1.106, "count": 207, "max_ms": 1.627, "p95_ms": 1.253}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.506, "count": 206, "max_ms": 42.562, "p95_ms": 35.275}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.642, "count": 206, "max_ms": 88.142, "p95_ms": 83.669}, "yolo11_640_fire_11class": {"avg_ms": 50.218, "count": 206, "max_ms": 68.321, "p95_ms": 64.296}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.145, "count": 206, "max_ms": 82.958, "p95_ms": 78.513}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.138, "count": 207, "max_ms": 160.499, "p95_ms": 155.493}, "read_interval": {"avg_ms": 145.466, "count": 206, "max_ms": 175.422, "p95_ms": 168.587}, "triton_wait": {"avg_ms": 63.798, "count": 206, "max_ms": 88.153, "p95_ms": 83.681}, "visualize": {"avg_ms": 4.359, "count": 206, "max_ms": 6.782, "p95_ms": 5.167}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10998.168, "limit_mb": 8796093022207.996}, "detail": {"pid": 89132, "restart_count": 485, "start_count": 486}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 443, "h264_decode_error_count": 1343, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:14:07+09:00", "last_error_summary": "[h264 @ 0x55e49b0ecf80] error while decoding MB 56 19, bytestream -12", "last_exit_at": "2026-07-01T08:14:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:14:10+09:00", "pid": 89132, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.082, "name": "ffmpeg", "pid": 89132, "rss_mb": 139.082, "state": "S (sleeping)", "threads": 106}, "restart_count": 485, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 486}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.186, "sample_count": 2013, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1393925, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:14:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.875, "last_frame_pushed_at": "2026-07-01T08:14:07+09:00", "last_frame_read_age_sec": 5.026, "last_frame_read_at": "2026-07-01T08:14:07+09:00", "mjpeg": {"age_sec": 4.858, "exists": true, "mtime": "2026-07-01T08:14:07.315334+09:00", "mtime_epoch": 1782861247.3153336, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171390}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:14:12+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5030.227, "name": "celery", "pid": 267, "rss_mb": 5030.227, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 171, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.752, "pushed": 5.752, "read": 5.719}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.531, "count": 172, "max_ms": 38.823, "p95_ms": 36.465}, "gstreamer_push": {"avg_ms": 1.115, "count": 172, "max_ms": 1.627, "p95_ms": 1.315}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.609, "count": 172, "max_ms": 42.562, "p95_ms": 35.848}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.283, "count": 172, "max_ms": 88.142, "p95_ms": 83.647}, "yolo11_640_fire_11class": {"avg_ms": 50.863, "count": 172, "max_ms": 68.321, "p95_ms": 64.729}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.969, "count": 172, "max_ms": 82.958, "p95_ms": 79.107}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.796, "count": 172, "max_ms": 160.499, "p95_ms": 155.917}, "read_interval": {"avg_ms": 146.175, "count": 171, "max_ms": 175.422, "p95_ms": 169.687}, "triton_wait": {"avg_ms": 64.439, "count": 172, "max_ms": 88.153, "p95_ms": 83.659}, "visualize": {"avg_ms": 4.372, "count": 172, "max_ms": 6.782, "p95_ms": 5.249}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10838.297, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 444, "h264_decode_error_count": 1344, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "last_exit_at": "2026-07-01T08:14:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.395, "last_started_at": "2026-07-01T08:14:10+09:00", "pid": 89132, "process": {"exists": false, "pid": 89132}, "restart_count": 485, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 486}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.941, "sample_count": 1982, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1394881, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T08:16:40+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T08:16:40+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T08:16:40.521739+09:00", "mtime_epoch": 1782861400.521739, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173936}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:16:40+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5030.742, "name": "celery", "pid": 267, "rss_mb": 5030.742, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.776, "pushed": 6.776, "read": 6.776}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.834, "count": 203, "max_ms": 38.031, "p95_ms": 36.572}, "gstreamer_push": {"avg_ms": 1.116, "count": 203, "max_ms": 1.559, "p95_ms": 1.277}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.151, "count": 203, "max_ms": 40.809, "p95_ms": 33.576}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.859, "count": 203, "max_ms": 90.851, "p95_ms": 84.753}, "yolo11_640_fire_11class": {"avg_ms": 49.742, "count": 203, "max_ms": 66.805, "p95_ms": 62.926}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.762, "count": 203, "max_ms": 84.354, "p95_ms": 79.754}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.437, "count": 203, "max_ms": 159.938, "p95_ms": 155.439}, "read_interval": {"avg_ms": 148.071, "count": 203, "max_ms": 171.795, "p95_ms": 168.481}, "triton_wait": {"avg_ms": 65.22, "count": 203, "max_ms": 90.864, "p95_ms": 84.765}, "visualize": {"avg_ms": 4.331, "count": 203, "max_ms": 5.997, "p95_ms": 4.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10838.023, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 444, "h264_decode_error_count": 1344, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "last_exit_at": "2026-07-01T08:16:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.48, "last_started_at": "2026-07-01T08:14:10+09:00", "pid": 89132, "process": {"exists": false, "pid": 89132}, "restart_count": 485, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 486}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.941, "sample_count": 1983, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1394882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:16:40+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-01T08:16:40+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-01T08:16:40.521739+09:00", "mtime_epoch": 1782861400.521739, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173936}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:16:40+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5030.742, "name": "celery", "pid": 267, "rss_mb": 5030.742, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.79, "pushed": 6.79, "read": 6.756}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.835, "count": 203, "max_ms": 38.031, "p95_ms": 36.572}, "gstreamer_push": {"avg_ms": 1.116, "count": 203, "max_ms": 1.559, "p95_ms": 1.277}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.145, "count": 203, "max_ms": 40.809, "p95_ms": 33.576}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.754, "count": 203, "max_ms": 90.851, "p95_ms": 84.753}, "yolo11_640_fire_11class": {"avg_ms": 49.664, "count": 203, "max_ms": 66.805, "p95_ms": 62.926}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.617, "count": 203, "max_ms": 84.354, "p95_ms": 79.654}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.327, "count": 203, "max_ms": 159.938, "p95_ms": 155.439}, "read_interval": {"avg_ms": 147.978, "count": 202, "max_ms": 171.795, "p95_ms": 168.481}, "triton_wait": {"avg_ms": 65.116, "count": 203, "max_ms": 90.864, "p95_ms": 84.765}, "visualize": {"avg_ms": 4.331, "count": 203, "max_ms": 5.997, "p95_ms": 4.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10837.945, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 444, "h264_decode_error_count": 1344, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "last_exit_at": "2026-07-01T08:16:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.485, "last_started_at": "2026-07-01T08:14:10+09:00", "pid": 89132, "process": {"exists": false, "pid": 89132}, "restart_count": 485, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 486}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.941, "sample_count": 1983, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1394882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:16:40+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T08:16:40+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-01T08:16:40.521739+09:00", "mtime_epoch": 1782861400.521739, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173936}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:16:40+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5030.742, "name": "celery", "pid": 267, "rss_mb": 5030.742, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.789, "pushed": 6.789, "read": 6.755}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.835, "count": 203, "max_ms": 38.031, "p95_ms": 36.572}, "gstreamer_push": {"avg_ms": 1.116, "count": 203, "max_ms": 1.559, "p95_ms": 1.277}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.145, "count": 203, "max_ms": 40.809, "p95_ms": 33.576}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.754, "count": 203, "max_ms": 90.851, "p95_ms": 84.753}, "yolo11_640_fire_11class": {"avg_ms": 49.664, "count": 203, "max_ms": 66.805, "p95_ms": 62.926}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.617, "count": 203, "max_ms": 84.354, "p95_ms": 79.654}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.327, "count": 203, "max_ms": 159.938, "p95_ms": 155.439}, "read_interval": {"avg_ms": 147.978, "count": 202, "max_ms": 171.795, "p95_ms": 168.481}, "triton_wait": {"avg_ms": 65.116, "count": 203, "max_ms": 90.864, "p95_ms": 84.765}, "visualize": {"avg_ms": 4.331, "count": 203, "max_ms": 5.997, "p95_ms": 4.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10935.18, "limit_mb": 8796093022207.996}, "detail": {"pid": 89299, "restart_count": 486, "start_count": 487}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 444, "h264_decode_error_count": 1344, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "last_exit_at": "2026-07-01T08:16:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:16:43+09:00", "pid": 89299, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.004, "name": "ffmpeg", "pid": 89299, "rss_mb": 136.004, "state": "S (sleeping)", "threads": 106}, "restart_count": 486, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 487}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.122, "sample_count": 1947, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1394882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.936, "last_frame_pushed_at": "2026-07-01T08:16:40+09:00", "last_frame_read_age_sec": 5.068, "last_frame_read_at": "2026-07-01T08:16:40+09:00", "mjpeg": {"age_sec": 4.919, "exists": true, "mtime": "2026-07-01T08:16:40.665741+09:00", "mtime_epoch": 1782861400.6657412, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:16:45+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5030.742, "name": "celery", "pid": 267, "rss_mb": 5030.742, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 171, "pushed_frames": 171, "read_frames": 170, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.704, "pushed": 5.704, "read": 5.671}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.946, "count": 170, "max_ms": 38.031, "p95_ms": 36.699}, "gstreamer_push": {"avg_ms": 1.117, "count": 171, "max_ms": 1.559, "p95_ms": 1.3}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.873, "count": 170, "max_ms": 40.809, "p95_ms": 33.576}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.068, "count": 170, "max_ms": 89.116, "p95_ms": 84.43}, "yolo11_640_fire_11class": {"avg_ms": 49.45, "count": 170, "max_ms": 66.288, "p95_ms": 62.599}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.249, "count": 170, "max_ms": 84.354, "p95_ms": 78.827}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.743, "count": 171, "max_ms": 157.379, "p95_ms": 152.643}, "read_interval": {"avg_ms": 147.265, "count": 170, "max_ms": 171.622, "p95_ms": 167.438}, "triton_wait": {"avg_ms": 64.497, "count": 170, "max_ms": 89.128, "p95_ms": 84.442}, "visualize": {"avg_ms": 4.341, "count": 170, "max_ms": 5.997, "p95_ms": 5.06}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11633}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10793.984, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 444, "h264_decode_error_count": 1344, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "last_exit_at": "2026-07-01T08:19:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.436, "last_started_at": "2026-07-01T08:16:43+09:00", "pid": 89299, "process": {"exists": false, "pid": 89299}, "restart_count": 486, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 487}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.539, "sample_count": 1984, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1395991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:19:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:19:27+09:00", "last_frame_read_age_sec": 0.15, "last_frame_read_at": "2026-07-01T08:19:26+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-01T08:19:26.876412+09:00", "mtime_epoch": 1782861566.8764124, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174849}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:19:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5031.258, "name": "celery", "pid": 267, "rss_mb": 5031.258, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.884, "pushed": 6.884, "read": 6.851}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.65, "count": 206, "max_ms": 38.631, "p95_ms": 34.452}, "gstreamer_push": {"avg_ms": 1.099, "count": 206, "max_ms": 1.366, "p95_ms": 1.159}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.686, "count": 206, "max_ms": 43.01, "p95_ms": 34.875}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.496, "count": 206, "max_ms": 87.992, "p95_ms": 83.953}, "yolo11_640_fire_11class": {"avg_ms": 48.56, "count": 206, "max_ms": 70.261, "p95_ms": 62.107}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.016, "count": 206, "max_ms": 84.64, "p95_ms": 80.207}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.694, "count": 206, "max_ms": 157.604, "p95_ms": 153.872}, "read_interval": {"avg_ms": 145.858, "count": 205, "max_ms": 172.377, "p95_ms": 165.884}, "triton_wait": {"avg_ms": 63.682, "count": 206, "max_ms": 88.004, "p95_ms": 83.965}, "visualize": {"avg_ms": 4.277, "count": 206, "max_ms": 5.162, "p95_ms": 4.524}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11633}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10793.977, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 444, "h264_decode_error_count": 1344, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "last_exit_at": "2026-07-01T08:19:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.441, "last_started_at": "2026-07-01T08:16:43+09:00", "pid": 89299, "process": {"exists": false, "pid": 89299}, "restart_count": 486, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 487}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.539, "sample_count": 1984, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1395991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:19:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:19:27+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-01T08:19:26+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-01T08:19:26.876412+09:00", "mtime_epoch": 1782861566.8764124, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174849}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:19:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5031.258, "name": "celery", "pid": 267, "rss_mb": 5031.258, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.883, "pushed": 6.883, "read": 6.85}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.65, "count": 206, "max_ms": 38.631, "p95_ms": 34.452}, "gstreamer_push": {"avg_ms": 1.099, "count": 206, "max_ms": 1.366, "p95_ms": 1.159}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.686, "count": 206, "max_ms": 43.01, "p95_ms": 34.875}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.496, "count": 206, "max_ms": 87.992, "p95_ms": 83.953}, "yolo11_640_fire_11class": {"avg_ms": 48.56, "count": 206, "max_ms": 70.261, "p95_ms": 62.107}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.016, "count": 206, "max_ms": 84.64, "p95_ms": 80.207}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.694, "count": 206, "max_ms": 157.604, "p95_ms": 153.872}, "read_interval": {"avg_ms": 145.858, "count": 205, "max_ms": 172.377, "p95_ms": 165.884}, "triton_wait": {"avg_ms": 63.682, "count": 206, "max_ms": 88.004, "p95_ms": 83.965}, "visualize": {"avg_ms": 4.277, "count": 206, "max_ms": 5.162, "p95_ms": 4.524}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11633}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10905.152, "limit_mb": 8796093022207.996}, "detail": {"pid": 89406, "restart_count": 487, "start_count": 488}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 444, "h264_decode_error_count": 1344, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "last_exit_at": "2026-07-01T08:19:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:19:29+09:00", "pid": 89406, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.902, "name": "ffmpeg", "pid": 89406, "rss_mb": 141.902, "state": "S (sleeping)", "threads": 106}, "restart_count": 487, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 488}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.076, "sample_count": 1962, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1395991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:19:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.213, "last_frame_pushed_at": "2026-07-01T08:19:27+09:00", "last_frame_read_age_sec": 4.358, "last_frame_read_at": "2026-07-01T08:19:26+09:00", "mjpeg": {"age_sec": 4.195, "exists": true, "mtime": "2026-07-01T08:19:27.032415+09:00", "mtime_epoch": 1782861567.032415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175572}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:19:31+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5031.258, "name": "celery", "pid": 267, "rss_mb": 5031.258, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.92, "pushed": 5.92, "read": 5.887}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.634, "count": 177, "max_ms": 38.631, "p95_ms": 34.529}, "gstreamer_push": {"avg_ms": 1.1, "count": 177, "max_ms": 1.366, "p95_ms": 1.16}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.737, "count": 177, "max_ms": 43.01, "p95_ms": 36.06}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.542, "count": 177, "max_ms": 87.992, "p95_ms": 84.219}, "yolo11_640_fire_11class": {"avg_ms": 48.631, "count": 177, "max_ms": 70.261, "p95_ms": 62.543}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.137, "count": 177, "max_ms": 81.872, "p95_ms": 80.207}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.71, "count": 177, "max_ms": 157.604, "p95_ms": 153.897}, "read_interval": {"avg_ms": 145.882, "count": 176, "max_ms": 172.377, "p95_ms": 165.985}, "triton_wait": {"avg_ms": 63.756, "count": 177, "max_ms": 88.004, "p95_ms": 84.231}, "visualize": {"avg_ms": 4.272, "count": 177, "max_ms": 5.162, "p95_ms": 4.523}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11642}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10796.168, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 444, "h264_decode_error_count": 1344, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "last_exit_at": "2026-07-01T08:22:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.611, "last_started_at": "2026-07-01T08:19:29+09:00", "pid": 89406, "process": {"exists": false, "pid": 89406}, "restart_count": 487, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 488}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.374, "sample_count": 2037, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1397104, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:22:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:22:12+09:00", "last_frame_read_age_sec": 0.135, "last_frame_read_at": "2026-07-01T08:22:12+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-01T08:22:12.711124+09:00", "mtime_epoch": 1782861732.711124, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173291}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:22:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5031.773, "name": "celery", "pid": 267, "rss_mb": 5031.773, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.927, "pushed": 6.927, "read": 6.893}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.449, "count": 207, "max_ms": 37.01, "p95_ms": 34.192}, "gstreamer_push": {"avg_ms": 1.101, "count": 207, "max_ms": 1.262, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.824, "count": 207, "max_ms": 42.456, "p95_ms": 35.357}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.713, "count": 207, "max_ms": 87.812, "p95_ms": 82.214}, "yolo11_640_fire_11class": {"avg_ms": 48.842, "count": 207, "max_ms": 64.675, "p95_ms": 62.467}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.332, "count": 207, "max_ms": 85.336, "p95_ms": 78.862}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.815, "count": 207, "max_ms": 156.933, "p95_ms": 151.619}, "read_interval": {"avg_ms": 145.058, "count": 206, "max_ms": 168.778, "p95_ms": 163.497}, "triton_wait": {"avg_ms": 63.198, "count": 207, "max_ms": 87.824, "p95_ms": 82.689}, "visualize": {"avg_ms": 4.269, "count": 207, "max_ms": 4.955, "p95_ms": 4.575}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11642}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10796.16, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 444, "h264_decode_error_count": 1344, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "last_exit_at": "2026-07-01T08:22:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.617, "last_started_at": "2026-07-01T08:19:29+09:00", "pid": 89406, "process": {"exists": false, "pid": 89406}, "restart_count": 487, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 488}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.374, "sample_count": 2037, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1397104, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:22:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:22:12+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-01T08:22:12+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-01T08:22:12.711124+09:00", "mtime_epoch": 1782861732.711124, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173291}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:22:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5031.773, "name": "celery", "pid": 267, "rss_mb": 5031.773, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.926, "pushed": 6.926, "read": 6.892}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.449, "count": 207, "max_ms": 37.01, "p95_ms": 34.192}, "gstreamer_push": {"avg_ms": 1.101, "count": 207, "max_ms": 1.262, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.824, "count": 207, "max_ms": 42.456, "p95_ms": 35.357}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.713, "count": 207, "max_ms": 87.812, "p95_ms": 82.214}, "yolo11_640_fire_11class": {"avg_ms": 48.842, "count": 207, "max_ms": 64.675, "p95_ms": 62.467}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.332, "count": 207, "max_ms": 85.336, "p95_ms": 78.862}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.815, "count": 207, "max_ms": 156.933, "p95_ms": 151.619}, "read_interval": {"avg_ms": 145.058, "count": 206, "max_ms": 168.778, "p95_ms": 163.497}, "triton_wait": {"avg_ms": 63.198, "count": 207, "max_ms": 87.824, "p95_ms": 82.689}, "visualize": {"avg_ms": 4.269, "count": 207, "max_ms": 4.955, "p95_ms": 4.575}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11642}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10903.859, "limit_mb": 8796093022207.996}, "detail": {"pid": 89513, "restart_count": 488, "start_count": 489}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 444, "h264_decode_error_count": 1344, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:16:40+09:00", "last_error_summary": "[h264 @ 0x56217d0a0f40] error while decoding MB 1 42, bytestream -5", "last_exit_at": "2026-07-01T08:22:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:22:15+09:00", "pid": 89513, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.598, "name": "ffmpeg", "pid": 89513, "rss_mb": 139.598, "state": "S (sleeping)", "threads": 106}, "restart_count": 488, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 489}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.374, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1397104, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:22:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.173, "last_frame_pushed_at": "2026-07-01T08:22:12+09:00", "last_frame_read_age_sec": 4.303, "last_frame_read_at": "2026-07-01T08:22:12+09:00", "mjpeg": {"age_sec": 4.155, "exists": true, "mtime": "2026-07-01T08:22:12.851126+09:00", "mtime_epoch": 1782861732.8511262, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173452}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:22:17+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5031.773, "name": "celery", "pid": 267, "rss_mb": 5031.773, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.971, "pushed": 5.971, "read": 5.938}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.457, "count": 178, "max_ms": 37.01, "p95_ms": 34.257}, "gstreamer_push": {"avg_ms": 1.099, "count": 179, "max_ms": 1.262, "p95_ms": 1.152}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.832, "count": 178, "max_ms": 41.814, "p95_ms": 35.357}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.692, "count": 178, "max_ms": 86.8, "p95_ms": 82.214}, "yolo11_640_fire_11class": {"avg_ms": 48.988, "count": 178, "max_ms": 64.675, "p95_ms": 62.717}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.599, "count": 178, "max_ms": 85.336, "p95_ms": 79.09}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.729, "count": 179, "max_ms": 156.799, "p95_ms": 151.619}, "read_interval": {"avg_ms": 144.974, "count": 178, "max_ms": 168.778, "p95_ms": 163.766}, "triton_wait": {"avg_ms": 63.255, "count": 178, "max_ms": 86.812, "p95_ms": 82.689}, "visualize": {"avg_ms": 4.262, "count": 178, "max_ms": 4.951, "p95_ms": 4.59}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10798.133, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e7f2088a80] error while decoding MB 71 23, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 445, "h264_decode_error_count": 1345, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:24:58+09:00", "last_error_summary": "[h264 @ 0x55e7f2088a80] error while decoding MB 71 23, bytestream -12", "last_exit_at": "2026-07-01T08:22:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.602, "last_started_at": "2026-07-01T08:22:15+09:00", "pid": 89513, "process": {"exists": false, "pid": 89513}, "restart_count": 488, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 489}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.581, "sample_count": 2032, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1398214, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:24:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T08:24:58+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T08:24:58+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T08:24:58.561869+09:00", "mtime_epoch": 1782861898.5618691, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172758}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:24:58+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5032.289, "name": "celery", "pid": 267, "rss_mb": 5032.289, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.868, "pushed": 6.868, "read": 6.902}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.761, "count": 206, "max_ms": 43.418, "p95_ms": 36.702}, "gstreamer_push": {"avg_ms": 1.102, "count": 206, "max_ms": 1.644, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.619, "count": 206, "max_ms": 43.496, "p95_ms": 35.624}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.935, "count": 206, "max_ms": 85.979, "p95_ms": 84.472}, "yolo11_640_fire_11class": {"avg_ms": 48.183, "count": 206, "max_ms": 64.536, "p95_ms": 61.953}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.848, "count": 206, "max_ms": 84.406, "p95_ms": 80.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.001, "count": 206, "max_ms": 158.172, "p95_ms": 152.933}, "read_interval": {"avg_ms": 145.334, "count": 207, "max_ms": 171.246, "p95_ms": 165.914}, "triton_wait": {"avg_ms": 63.093, "count": 206, "max_ms": 85.992, "p95_ms": 84.484}, "visualize": {"avg_ms": 4.202, "count": 206, "max_ms": 5.253, "p95_ms": 4.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10797.734, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:24:58+09:00", "last_error_summary": "[h264 @ 0x55e7f2088a80] error while decoding MB 71 23, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 445, "h264_decode_error_count": 1345, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:24:58+09:00", "last_error_summary": "[h264 @ 0x55e7f2088a80] error while decoding MB 71 23, bytestream -12", "last_exit_at": "2026-07-01T08:24:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.708, "last_started_at": "2026-07-01T08:22:15+09:00", "pid": 89513, "process": {"exists": false, "pid": 89513}, "restart_count": 488, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 489}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.581, "sample_count": 2032, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1398215, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:24:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:24:58+09:00", "last_frame_read_age_sec": 0.158, "last_frame_read_at": "2026-07-01T08:24:58+09:00", "mjpeg": {"age_sec": 0.151, "exists": true, "mtime": "2026-07-01T08:24:58.561869+09:00", "mtime_epoch": 1782861898.5618691, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172758}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:24:58+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5032.289, "name": "celery", "pid": 267, "rss_mb": 5032.289, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.906, "pushed": 6.906, "read": 6.872}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.767, "count": 206, "max_ms": 43.418, "p95_ms": 36.702}, "gstreamer_push": {"avg_ms": 1.102, "count": 207, "max_ms": 1.644, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.597, "count": 206, "max_ms": 43.496, "p95_ms": 35.624}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.059, "count": 206, "max_ms": 85.979, "p95_ms": 84.472}, "yolo11_640_fire_11class": {"avg_ms": 48.244, "count": 206, "max_ms": 64.536, "p95_ms": 61.953}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.956, "count": 206, "max_ms": 84.406, "p95_ms": 80.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.09, "count": 207, "max_ms": 158.172, "p95_ms": 152.933}, "read_interval": {"avg_ms": 145.343, "count": 206, "max_ms": 171.246, "p95_ms": 165.914}, "triton_wait": {"avg_ms": 63.217, "count": 206, "max_ms": 85.992, "p95_ms": 84.484}, "visualize": {"avg_ms": 4.202, "count": 206, "max_ms": 5.253, "p95_ms": 4.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10797.727, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 445, "h264_decode_error_count": 1345, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:24:58+09:00", "last_error_summary": "[h264 @ 0x55e7f2088a80] error while decoding MB 71 23, bytestream -12", "last_exit_at": "2026-07-01T08:24:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.713, "last_started_at": "2026-07-01T08:22:15+09:00", "pid": 89513, "process": {"exists": false, "pid": 89513}, "restart_count": 488, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 489}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.581, "sample_count": 2032, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1398215, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:24:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:24:58+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-01T08:24:58+09:00", "mjpeg": {"age_sec": 0.156, "exists": true, "mtime": "2026-07-01T08:24:58.561869+09:00", "mtime_epoch": 1782861898.5618691, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172758}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:24:58+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5032.289, "name": "celery", "pid": 267, "rss_mb": 5032.289, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.904, "pushed": 6.904, "read": 6.871}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.767, "count": 206, "max_ms": 43.418, "p95_ms": 36.702}, "gstreamer_push": {"avg_ms": 1.102, "count": 207, "max_ms": 1.644, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.597, "count": 206, "max_ms": 43.496, "p95_ms": 35.624}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.059, "count": 206, "max_ms": 85.979, "p95_ms": 84.472}, "yolo11_640_fire_11class": {"avg_ms": 48.244, "count": 206, "max_ms": 64.536, "p95_ms": 61.953}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.956, "count": 206, "max_ms": 84.406, "p95_ms": 80.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.09, "count": 207, "max_ms": 158.172, "p95_ms": 152.933}, "read_interval": {"avg_ms": 145.343, "count": 206, "max_ms": 171.246, "p95_ms": 165.914}, "triton_wait": {"avg_ms": 63.217, "count": 206, "max_ms": 85.992, "p95_ms": 84.484}, "visualize": {"avg_ms": 4.202, "count": 206, "max_ms": 5.253, "p95_ms": 4.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10905.781, "limit_mb": 8796093022207.996}, "detail": {"pid": 89620, "restart_count": 489, "start_count": 490}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 445, "h264_decode_error_count": 1345, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:24:58+09:00", "last_error_summary": "[h264 @ 0x55e7f2088a80] error while decoding MB 71 23, bytestream -12", "last_exit_at": "2026-07-01T08:25:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:25:00+09:00", "pid": 89620, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.117, "name": "ffmpeg", "pid": 89620, "rss_mb": 139.117, "state": "S (sleeping)", "threads": 106}, "restart_count": 489, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 490}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.607, "sample_count": 2005, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1398215, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:24:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.037, "last_frame_pushed_at": "2026-07-01T08:24:58+09:00", "last_frame_read_age_sec": 4.19, "last_frame_read_at": "2026-07-01T08:24:58+09:00", "mjpeg": {"age_sec": 4.019, "exists": true, "mtime": "2026-07-01T08:24:58.725872+09:00", "mtime_epoch": 1782861898.7258718, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171535}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:25:02+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5032.289, "name": "celery", "pid": 267, "rss_mb": 5032.289, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.967, "pushed": 5.967, "read": 5.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.867, "count": 178, "max_ms": 43.418, "p95_ms": 40.88}, "gstreamer_push": {"avg_ms": 1.099, "count": 179, "max_ms": 1.223, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.431, "count": 178, "max_ms": 43.496, "p95_ms": 35.842}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.246, "count": 178, "max_ms": 85.979, "p95_ms": 84.739}, "yolo11_640_fire_11class": {"avg_ms": 48.409, "count": 178, "max_ms": 64.536, "p95_ms": 62.456}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.146, "count": 178, "max_ms": 84.406, "p95_ms": 81.049}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.467, "count": 179, "max_ms": 158.172, "p95_ms": 153.084}, "read_interval": {"avg_ms": 145.729, "count": 178, "max_ms": 171.246, "p95_ms": 166.269}, "triton_wait": {"avg_ms": 63.41, "count": 178, "max_ms": 85.992, "p95_ms": 84.752}, "visualize": {"avg_ms": 4.198, "count": 178, "max_ms": 5.253, "p95_ms": 4.558}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 67, "tracking_summary": 13149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10737.641, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x56351cd23200] missing picture in access unit with size 18 [h264 @ 0x56351ce69740] no frame! [in#0/rtsp @ 0x56351cd1ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56351cd1ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56351cd23840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 200, "h264_decode_error_count": 206, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:27:06+09:00", "last_error_summary": "[NULL @ 0x56351cd23200] missing picture in access unit with size 18 [h264 @ 0x56351ce69740] no frame! [in#0/rtsp @ 0x56351cd1ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56351cd1ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56351cd23840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T08:13:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 804.881, "last_started_at": "2026-07-01T08:13:41+09:00", "pid": 88980, "process": {"exists": false, "pid": 88980}, "restart_count": 199, "restart_count_per_hour": 1, "rtsp_timeout_count": 326, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 200}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.505, "sample_count": 2538, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1577928, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:27:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-01T08:27:06+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-01T08:27:06+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-01T08:27:06.700007+09:00", "mtime_epoch": 1782862026.7000072, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 213133}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:27:06+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1804.266, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 42, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.4, "pushed": 1.4, "read": 1.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.096, "count": 42, "max_ms": 29.895, "p95_ms": 22.724}, "gstreamer_push": {"avg_ms": 1.167, "count": 42, "max_ms": 1.34, "p95_ms": 1.278}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.641, "count": 42, "max_ms": 60.194, "p95_ms": 57.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.608, "count": 42, "max_ms": 82.37, "p95_ms": 80.249}, "yolo11_640_fire_11class": {"avg_ms": 48.737, "count": 42, "max_ms": 61.056, "p95_ms": 59.836}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.041, "count": 42, "max_ms": 81.22, "p95_ms": 79.107}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.587, "count": 42, "max_ms": 116.693, "p95_ms": 108.86}, "read_interval": {"avg_ms": 714.539, "count": 42, "max_ms": 25921.838, "p95_ms": 124.202}, "triton_wait": {"avg_ms": 58.005, "count": 42, "max_ms": 82.383, "p95_ms": 80.259}, "visualize": {"avg_ms": 3.379, "count": 42, "max_ms": 4.364, "p95_ms": 3.552}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 67, "tracking_summary": 13149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10737.203, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:27:06+09:00", "last_error_summary": "[NULL @ 0x56351cd23200] missing picture in access unit with size 18 [h264 @ 0x56351ce69740] no frame! [in#0/rtsp @ 0x56351cd1ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56351cd1ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56351cd23840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 200, "h264_decode_error_count": 206, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:27:06+09:00", "last_error_summary": "[NULL @ 0x56351cd23200] missing picture in access unit with size 18 [h264 @ 0x56351ce69740] no frame! [in#0/rtsp @ 0x56351cd1ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56351cd1ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56351cd23840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T08:27:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 804.935, "last_started_at": "2026-07-01T08:13:41+09:00", "pid": 88980, "process": {"exists": false, "pid": 88980}, "restart_count": 199, "restart_count_per_hour": 1, "rtsp_timeout_count": 326, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 200}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.505, "sample_count": 2539, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1577929, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:27:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T08:27:06+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-01T08:27:06+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-01T08:27:06.700007+09:00", "mtime_epoch": 1782862026.7000072, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 213133}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:27:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1804.266, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 41, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.402, "pushed": 1.402, "read": 1.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.13, "count": 42, "max_ms": 29.895, "p95_ms": 22.724}, "gstreamer_push": {"avg_ms": 1.17, "count": 42, "max_ms": 1.34, "p95_ms": 1.278}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.766, "count": 42, "max_ms": 60.194, "p95_ms": 57.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.169, "count": 42, "max_ms": 82.37, "p95_ms": 80.249}, "yolo11_640_fire_11class": {"avg_ms": 49.029, "count": 42, "max_ms": 61.056, "p95_ms": 59.836}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.344, "count": 42, "max_ms": 81.22, "p95_ms": 79.107}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.184, "count": 42, "max_ms": 116.693, "p95_ms": 108.86}, "read_interval": {"avg_ms": 729.709, "count": 41, "max_ms": 25921.838, "p95_ms": 124.202}, "triton_wait": {"avg_ms": 58.567, "count": 42, "max_ms": 82.383, "p95_ms": 80.259}, "visualize": {"avg_ms": 3.376, "count": 42, "max_ms": 4.364, "p95_ms": 3.549}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 67, "tracking_summary": 13149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10737.195, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 200, "h264_decode_error_count": 206, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:27:06+09:00", "last_error_summary": "[NULL @ 0x56351cd23200] missing picture in access unit with size 18 [h264 @ 0x56351ce69740] no frame! [in#0/rtsp @ 0x56351cd1ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56351cd1ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56351cd23840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T08:27:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 804.941, "last_started_at": "2026-07-01T08:13:41+09:00", "pid": 88980, "process": {"exists": false, "pid": 88980}, "restart_count": 199, "restart_count_per_hour": 1, "rtsp_timeout_count": 326, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 200}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.505, "sample_count": 2539, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1577929, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:27:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T08:27:06+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-01T08:27:06+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-01T08:27:06.700007+09:00", "mtime_epoch": 1782862026.7000072, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 213133}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:27:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1804.266, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 41, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.402, "pushed": 1.402, "read": 1.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.13, "count": 42, "max_ms": 29.895, "p95_ms": 22.724}, "gstreamer_push": {"avg_ms": 1.17, "count": 42, "max_ms": 1.34, "p95_ms": 1.278}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.766, "count": 42, "max_ms": 60.194, "p95_ms": 57.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.169, "count": 42, "max_ms": 82.37, "p95_ms": 80.249}, "yolo11_640_fire_11class": {"avg_ms": 49.029, "count": 42, "max_ms": 61.056, "p95_ms": 59.836}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.344, "count": 42, "max_ms": 81.22, "p95_ms": 79.107}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.184, "count": 42, "max_ms": 116.693, "p95_ms": 108.86}, "read_interval": {"avg_ms": 729.709, "count": 41, "max_ms": 25921.838, "p95_ms": 124.202}, "triton_wait": {"avg_ms": 58.567, "count": 42, "max_ms": 82.383, "p95_ms": 80.259}, "visualize": {"avg_ms": 3.376, "count": 42, "max_ms": 4.364, "p95_ms": 3.549}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 67, "tracking_summary": 13149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10761.359, "limit_mb": 8796093022207.996}, "detail": {"pid": 89787, "restart_count": 200, "start_count": 201}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 200, "h264_decode_error_count": 206, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:27:06+09:00", "last_error_summary": "[NULL @ 0x56351cd23200] missing picture in access unit with size 18 [h264 @ 0x56351ce69740] no frame! [in#0/rtsp @ 0x56351cd1ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56351cd1ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x56351cd23840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T08:27:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:27:08+09:00", "pid": 89787, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.172, "name": "ffmpeg", "pid": 89787, "rss_mb": 57.172, "state": "S (sleeping)", "threads": 1}, "restart_count": 200, "restart_count_per_hour": 2, "rtsp_timeout_count": 326, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 201}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.606, "sample_count": 2502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1577929, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:27:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.077, "last_frame_pushed_at": "2026-07-01T08:27:06+09:00", "last_frame_read_age_sec": 4.182, "last_frame_read_at": "2026-07-01T08:27:06+09:00", "mjpeg": {"age_sec": 4.063, "exists": true, "mtime": "2026-07-01T08:27:06.816009+09:00", "mtime_epoch": 1782862026.816009, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211972}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:27:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1804.266, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.265, "pushed": 2.265, "read": 2.265}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.836, "count": 12, "max_ms": 28.362, "p95_ms": 28.362}, "gstreamer_push": {"avg_ms": 1.156, "count": 12, "max_ms": 1.228, "p95_ms": 1.228}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.6, "count": 12, "max_ms": 57.098, "p95_ms": 57.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.942, "count": 12, "max_ms": 79.825, "p95_ms": 79.825}, "yolo11_640_fire_11class": {"avg_ms": 49.483, "count": 12, "max_ms": 58.089, "p95_ms": 58.089}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.493, "count": 12, "max_ms": 79.107, "p95_ms": 79.107}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.926, "count": 12, "max_ms": 116.693, "p95_ms": 116.693}, "read_interval": {"avg_ms": 2253.15, "count": 12, "max_ms": 25921.838, "p95_ms": 25921.838}, "triton_wait": {"avg_ms": 60.125, "count": 12, "max_ms": 79.838, "p95_ms": 79.838}, "visualize": {"avg_ms": 3.46, "count": 12, "max_ms": 4.364, "p95_ms": 4.364}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.298, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11660}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10799.797, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:24:58+09:00", "last_error_summary": "[h264 @ 0x55e7f2088a80] error while decoding MB 71 23, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 445, "h264_decode_error_count": 1345, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:24:58+09:00", "last_error_summary": "[h264 @ 0x55e7f2088a80] error while decoding MB 71 23, bytestream -12", "last_exit_at": "2026-07-01T08:27:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.037, "last_started_at": "2026-07-01T08:25:00+09:00", "pid": 89620, "process": {"exists": false, "pid": 89620}, "restart_count": 489, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 490}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.589, "sample_count": 2028, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1399316, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:27:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T08:27:43+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-01T08:27:43+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-01T08:27:43.648626+09:00", "mtime_epoch": 1782862063.6486259, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170337}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:27:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5032.805, "name": "celery", "pid": 267, "rss_mb": 5032.805, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.176, "pushed": 7.176, "read": 7.143}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.644, "count": 215, "max_ms": 45.336, "p95_ms": 35.617}, "gstreamer_push": {"avg_ms": 1.036, "count": 215, "max_ms": 1.385, "p95_ms": 1.135}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.674, "count": 215, "max_ms": 44.54, "p95_ms": 38.114}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.262, "count": 215, "max_ms": 86.667, "p95_ms": 75.266}, "yolo11_640_fire_11class": {"avg_ms": 45.597, "count": 215, "max_ms": 62.127, "p95_ms": 54.642}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.997, "count": 215, "max_ms": 81.822, "p95_ms": 73.978}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.022, "count": 215, "max_ms": 155.577, "p95_ms": 145.539}, "read_interval": {"avg_ms": 139.927, "count": 214, "max_ms": 170.159, "p95_ms": 158.079}, "triton_wait": {"avg_ms": 58.403, "count": 215, "max_ms": 86.679, "p95_ms": 75.964}, "visualize": {"avg_ms": 4.231, "count": 215, "max_ms": 5.676, "p95_ms": 4.691}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11660}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10799.789, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 445, "h264_decode_error_count": 1345, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:24:58+09:00", "last_error_summary": "[h264 @ 0x55e7f2088a80] error while decoding MB 71 23, bytestream -12", "last_exit_at": "2026-07-01T08:27:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.042, "last_started_at": "2026-07-01T08:25:00+09:00", "pid": 89620, "process": {"exists": false, "pid": 89620}, "restart_count": 489, "restart_count_per_hour": 21, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 490}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.589, "sample_count": 2028, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1399316, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:27:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T08:27:43+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-01T08:27:43+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-01T08:27:43.648626+09:00", "mtime_epoch": 1782862063.6486259, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170337}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:27:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5032.805, "name": "celery", "pid": 267, "rss_mb": 5032.805, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.175, "pushed": 7.175, "read": 7.142}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.644, "count": 215, "max_ms": 45.336, "p95_ms": 35.617}, "gstreamer_push": {"avg_ms": 1.036, "count": 215, "max_ms": 1.385, "p95_ms": 1.135}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.674, "count": 215, "max_ms": 44.54, "p95_ms": 38.114}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.262, "count": 215, "max_ms": 86.667, "p95_ms": 75.266}, "yolo11_640_fire_11class": {"avg_ms": 45.597, "count": 215, "max_ms": 62.127, "p95_ms": 54.642}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.997, "count": 215, "max_ms": 81.822, "p95_ms": 73.978}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.022, "count": 215, "max_ms": 155.577, "p95_ms": 145.539}, "read_interval": {"avg_ms": 139.927, "count": 214, "max_ms": 170.159, "p95_ms": 158.079}, "triton_wait": {"avg_ms": 58.403, "count": 215, "max_ms": 86.679, "p95_ms": 75.964}, "visualize": {"avg_ms": 4.231, "count": 215, "max_ms": 5.676, "p95_ms": 4.691}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11660}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11099.52, "limit_mb": 8796093022207.996}, "detail": {"pid": 89999, "restart_count": 490, "start_count": 491}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 445, "h264_decode_error_count": 1345, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:24:58+09:00", "last_error_summary": "[h264 @ 0x55e7f2088a80] error while decoding MB 71 23, bytestream -12", "last_exit_at": "2026-07-01T08:27:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:27:46+09:00", "pid": 89999, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.434, "name": "ffmpeg", "pid": 89999, "rss_mb": 139.434, "state": "S (sleeping)", "threads": 106}, "restart_count": 490, "restart_count_per_hour": 22, "rtsp_timeout_count": 392, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 491}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.607, "sample_count": 1997, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1399316, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:27:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.753, "last_frame_pushed_at": "2026-07-01T08:27:43+09:00", "last_frame_read_age_sec": 4.884, "last_frame_read_at": "2026-07-01T08:27:43+09:00", "mjpeg": {"age_sec": 4.74, "exists": true, "mtime": "2026-07-01T08:27:43.788628+09:00", "mtime_epoch": 1782862063.7886283, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170534}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:27:48+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5032.805, "name": "celery", "pid": 267, "rss_mb": 5032.805, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.0, "pushed": 6.0, "read": 6.0}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.583, "count": 180, "max_ms": 44.723, "p95_ms": 35.002}, "gstreamer_push": {"avg_ms": 1.034, "count": 180, "max_ms": 1.221, "p95_ms": 1.135}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.645, "count": 180, "max_ms": 44.54, "p95_ms": 38.114}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.817, "count": 180, "max_ms": 86.667, "p95_ms": 75.482}, "yolo11_640_fire_11class": {"avg_ms": 45.574, "count": 180, "max_ms": 62.127, "p95_ms": 54.432}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.988, "count": 180, "max_ms": 81.822, "p95_ms": 71.016}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.422, "count": 180, "max_ms": 155.577, "p95_ms": 145.539}, "read_interval": {"avg_ms": 140.288, "count": 180, "max_ms": 170.159, "p95_ms": 158.079}, "triton_wait": {"avg_ms": 58.857, "count": 180, "max_ms": 86.679, "p95_ms": 75.494}, "visualize": {"avg_ms": 4.228, "count": 180, "max_ms": 5.081, "p95_ms": 4.657}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11669}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10810.953, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 10, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5579c2966740] cabac decode of qscale diff failed at 33 39 [h264 @ 0x5579c2966740] error while decoding MB 33 39, bytestream 1479 [h264 @ 0x5579c3358140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579c3358140] error while decoding MB 0 43, bytestream 4030 [h264 @ 0x5579c2979b80] left block unavailable for requested intra mode [h264 @ 0x5579c2979b80] error while decoding MB 0 35, bytestream 12652 [h264 @ 0x5579c28d7b00] left block unavailable for requested intra4x4 ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 446, "h264_decode_error_count": 1355, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:30:36+09:00", "last_error_summary": "[h264 @ 0x5579c2966740] cabac decode of qscale diff failed at 33 39 [h264 @ 0x5579c2966740] error while decoding MB 33 39, bytestream 1479 [h264 @ 0x5579c3358140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579c3358140] error while decoding MB 0 43, bytestream 4030 [h264 @ 0x5579c2979b80] left block unavailable for requested intra mode [h264 @ 0x5579c2979b80] error while decoding MB 0 35, bytestream 12652 [h264 @ 0x5579c28d7b00] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-01T08:27:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 169.784, "last_started_at": "2026-07-01T08:27:46+09:00", "pid": 89999, "process": {"exists": false, "pid": 89999}, "restart_count": 490, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 491}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.644, "sample_count": 1929, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1400370, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:30:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T08:30:36+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T08:30:36+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T08:30:36.271527+09:00", "mtime_epoch": 1782862236.271527, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172189}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:30:36+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5033.32, "name": "celery", "pid": 267, "rss_mb": 5033.32, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 81, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.703, "pushed": 2.703, "read": 2.703}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 29.831, "count": 81, "max_ms": 34.872, "p95_ms": 33.758}, "gstreamer_push": {"avg_ms": 1.072, "count": 81, "max_ms": 1.177, "p95_ms": 1.138}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.485, "count": 81, "max_ms": 41.042, "p95_ms": 35.062}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.306, "count": 81, "max_ms": 86.457, "p95_ms": 83.061}, "yolo11_640_fire_11class": {"avg_ms": 46.959, "count": 81, "max_ms": 63.795, "p95_ms": 61.825}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.025, "count": 81, "max_ms": 84.937, "p95_ms": 79.603}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.721, "count": 81, "max_ms": 158.734, "p95_ms": 153.514}, "read_interval": {"avg_ms": 370.834, "count": 81, "max_ms": 10132.189, "p95_ms": 167.423}, "triton_wait": {"avg_ms": 59.384, "count": 81, "max_ms": 86.47, "p95_ms": 83.072}, "visualize": {"avg_ms": 4.231, "count": 81, "max_ms": 5.097, "p95_ms": 4.477}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11669}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10810.828, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:30:36+09:00", "last_error_summary": "[h264 @ 0x5579c2966740] cabac decode of qscale diff failed at 33 39 [h264 @ 0x5579c2966740] error while decoding MB 33 39, bytestream 1479 [h264 @ 0x5579c3358140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579c3358140] error while decoding MB 0 43, bytestream 4030 [h264 @ 0x5579c2979b80] left block unavailable for requested intra mode [h264 @ 0x5579c2979b80] error while decoding MB 0 35, bytestream 12652 [h264 @ 0x5579c28d7b00] left block unavailable for requested intra4x4 ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 446, "h264_decode_error_count": 1355, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:30:36+09:00", "last_error_summary": "[h264 @ 0x5579c2966740] cabac decode of qscale diff failed at 33 39 [h264 @ 0x5579c2966740] error while decoding MB 33 39, bytestream 1479 [h264 @ 0x5579c3358140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579c3358140] error while decoding MB 0 43, bytestream 4030 [h264 @ 0x5579c2979b80] left block unavailable for requested intra mode [h264 @ 0x5579c2979b80] error while decoding MB 0 35, bytestream 12652 [h264 @ 0x5579c28d7b00] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-01T08:30:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 169.876, "last_started_at": "2026-07-01T08:27:46+09:00", "pid": 89999, "process": {"exists": false, "pid": 89999}, "restart_count": 490, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 491}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.644, "sample_count": 1929, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1400371, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:30:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T08:30:36+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-01T08:30:36+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-01T08:30:36.271527+09:00", "mtime_epoch": 1782862236.271527, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172189}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:30:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5033.32, "name": "celery", "pid": 267, "rss_mb": 5033.32, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.707, "pushed": 2.707, "read": 2.674}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 29.85, "count": 81, "max_ms": 35.094, "p95_ms": 33.85}, "gstreamer_push": {"avg_ms": 1.072, "count": 81, "max_ms": 1.177, "p95_ms": 1.138}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.454, "count": 81, "max_ms": 41.042, "p95_ms": 35.062}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.32, "count": 81, "max_ms": 86.457, "p95_ms": 83.061}, "yolo11_640_fire_11class": {"avg_ms": 46.914, "count": 81, "max_ms": 63.795, "p95_ms": 61.825}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.996, "count": 81, "max_ms": 84.937, "p95_ms": 79.603}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.826, "count": 81, "max_ms": 158.734, "p95_ms": 153.514}, "read_interval": {"avg_ms": 373.732, "count": 80, "max_ms": 10132.189, "p95_ms": 167.423}, "triton_wait": {"avg_ms": 59.398, "count": 81, "max_ms": 86.47, "p95_ms": 83.072}, "visualize": {"avg_ms": 4.239, "count": 81, "max_ms": 5.097, "p95_ms": 4.485}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11669}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10810.816, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 446, "h264_decode_error_count": 1355, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:30:36+09:00", "last_error_summary": "[h264 @ 0x5579c2966740] cabac decode of qscale diff failed at 33 39 [h264 @ 0x5579c2966740] error while decoding MB 33 39, bytestream 1479 [h264 @ 0x5579c3358140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579c3358140] error while decoding MB 0 43, bytestream 4030 [h264 @ 0x5579c2979b80] left block unavailable for requested intra mode [h264 @ 0x5579c2979b80] error while decoding MB 0 35, bytestream 12652 [h264 @ 0x5579c28d7b00] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-01T08:30:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 169.881, "last_started_at": "2026-07-01T08:27:46+09:00", "pid": 89999, "process": {"exists": false, "pid": 89999}, "restart_count": 490, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 491}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.644, "sample_count": 1929, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1400371, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:30:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:30:36+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-01T08:30:36+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-01T08:30:36.271527+09:00", "mtime_epoch": 1782862236.271527, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172189}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:30:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5033.32, "name": "celery", "pid": 267, "rss_mb": 5033.32, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.707, "pushed": 2.707, "read": 2.673}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 29.85, "count": 81, "max_ms": 35.094, "p95_ms": 33.85}, "gstreamer_push": {"avg_ms": 1.072, "count": 81, "max_ms": 1.177, "p95_ms": 1.138}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.454, "count": 81, "max_ms": 41.042, "p95_ms": 35.062}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.32, "count": 81, "max_ms": 86.457, "p95_ms": 83.061}, "yolo11_640_fire_11class": {"avg_ms": 46.914, "count": 81, "max_ms": 63.795, "p95_ms": 61.825}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.996, "count": 81, "max_ms": 84.937, "p95_ms": 79.603}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.826, "count": 81, "max_ms": 158.734, "p95_ms": 153.514}, "read_interval": {"avg_ms": 373.732, "count": 80, "max_ms": 10132.189, "p95_ms": 167.423}, "triton_wait": {"avg_ms": 59.398, "count": 81, "max_ms": 86.47, "p95_ms": 83.072}, "visualize": {"avg_ms": 4.239, "count": 81, "max_ms": 5.097, "p95_ms": 4.485}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13161}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10505.453, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55af92ed6c80] error while decoding MB 47 39, bytestream -23 [in#0/rtsp @ 0x55af925d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af925d7e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 201, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:30:38+09:00", "last_error_summary": "[h264 @ 0x55af92ed6c80] error while decoding MB 47 39, bytestream -23 [in#0/rtsp @ 0x55af925d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af925d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:27:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.703, "last_started_at": "2026-07-01T08:27:08+09:00", "pid": 89787, "process": {"exists": false, "pid": 89787}, "restart_count": 200, "restart_count_per_hour": 2, "rtsp_timeout_count": 328, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 201}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 191.415, "sample_count": 1968, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1579407, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:30:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-01T08:30:38+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-01T08:30:38+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T08:30:38.519565+09:00", "mtime_epoch": 1782862238.5195649, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 201593}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:30:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1805.109, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.134, "pushed": 2.134, "read": 2.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.904, "count": 63, "max_ms": 25.693, "p95_ms": 22.27}, "gstreamer_push": {"avg_ms": 1.188, "count": 64, "max_ms": 1.423, "p95_ms": 1.295}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.787, "count": 63, "max_ms": 60.611, "p95_ms": 55.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.306, "count": 63, "max_ms": 83.59, "p95_ms": 78.561}, "yolo11_640_fire_11class": {"avg_ms": 48.687, "count": 63, "max_ms": 63.217, "p95_ms": 60.073}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.3, "count": 63, "max_ms": 80.432, "p95_ms": 77.938}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.616, "count": 64, "max_ms": 139.423, "p95_ms": 109.334}, "read_interval": {"avg_ms": 468.888, "count": 64, "max_ms": 13505.823, "p95_ms": 126.201}, "triton_wait": {"avg_ms": 58.791, "count": 63, "max_ms": 83.6, "p95_ms": 78.92}, "visualize": {"avg_ms": 3.447, "count": 63, "max_ms": 4.92, "p95_ms": 3.715}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13161}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10505.312, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:30:38+09:00", "last_error_summary": "[h264 @ 0x55af92ed6c80] error while decoding MB 47 39, bytestream -23 [in#0/rtsp @ 0x55af925d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af925d7e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 201, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:30:38+09:00", "last_error_summary": "[h264 @ 0x55af92ed6c80] error while decoding MB 47 39, bytestream -23 [in#0/rtsp @ 0x55af925d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af925d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:30:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.729, "last_started_at": "2026-07-01T08:27:08+09:00", "pid": 89787, "process": {"exists": false, "pid": 89787}, "restart_count": 200, "restart_count_per_hour": 2, "rtsp_timeout_count": 328, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 201}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 191.415, "sample_count": 1968, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1579408, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:30:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T08:30:38+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-01T08:30:38+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-01T08:30:38.519565+09:00", "mtime_epoch": 1782862238.5195649, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 201593}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:30:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1805.109, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.139, "pushed": 2.139, "read": 2.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.932, "count": 64, "max_ms": 25.693, "p95_ms": 22.27}, "gstreamer_push": {"avg_ms": 1.192, "count": 64, "max_ms": 1.423, "p95_ms": 1.317}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.726, "count": 64, "max_ms": 60.611, "p95_ms": 55.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.204, "count": 64, "max_ms": 83.59, "p95_ms": 78.561}, "yolo11_640_fire_11class": {"avg_ms": 48.642, "count": 64, "max_ms": 63.217, "p95_ms": 60.073}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.208, "count": 64, "max_ms": 80.432, "p95_ms": 77.938}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.672, "count": 64, "max_ms": 139.423, "p95_ms": 109.334}, "read_interval": {"avg_ms": 474.833, "count": 63, "max_ms": 13505.823, "p95_ms": 126.201}, "triton_wait": {"avg_ms": 58.681, "count": 64, "max_ms": 83.6, "p95_ms": 78.92}, "visualize": {"avg_ms": 3.488, "count": 64, "max_ms": 6.094, "p95_ms": 4.539}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13161}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10505.273, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 201, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:30:38+09:00", "last_error_summary": "[h264 @ 0x55af92ed6c80] error while decoding MB 47 39, bytestream -23 [in#0/rtsp @ 0x55af925d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af925d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:30:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 209.736, "last_started_at": "2026-07-01T08:27:08+09:00", "pid": 89787, "process": {"exists": false, "pid": 89787}, "restart_count": 200, "restart_count_per_hour": 2, "rtsp_timeout_count": 328, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 201}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 191.415, "sample_count": 1968, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1579408, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:30:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T08:30:38+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T08:30:38+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-01T08:30:38.519565+09:00", "mtime_epoch": 1782862238.5195649, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 201593}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:30:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1805.109, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.138, "pushed": 2.138, "read": 2.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.932, "count": 64, "max_ms": 25.693, "p95_ms": 22.27}, "gstreamer_push": {"avg_ms": 1.192, "count": 64, "max_ms": 1.423, "p95_ms": 1.317}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.726, "count": 64, "max_ms": 60.611, "p95_ms": 55.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.204, "count": 64, "max_ms": 83.59, "p95_ms": 78.561}, "yolo11_640_fire_11class": {"avg_ms": 48.642, "count": 64, "max_ms": 63.217, "p95_ms": 60.073}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.208, "count": 64, "max_ms": 80.432, "p95_ms": 77.938}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.672, "count": 64, "max_ms": 139.423, "p95_ms": 109.334}, "read_interval": {"avg_ms": 474.833, "count": 63, "max_ms": 13505.823, "p95_ms": 126.201}, "triton_wait": {"avg_ms": 58.681, "count": 64, "max_ms": 83.6, "p95_ms": 78.92}, "visualize": {"avg_ms": 3.488, "count": 64, "max_ms": 6.094, "p95_ms": 4.539}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11669}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10620.172, "limit_mb": 8796093022207.996}, "detail": {"pid": 90106, "restart_count": 491, "start_count": 492}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 446, "h264_decode_error_count": 1355, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:30:36+09:00", "last_error_summary": "[h264 @ 0x5579c2966740] cabac decode of qscale diff failed at 33 39 [h264 @ 0x5579c2966740] error while decoding MB 33 39, bytestream 1479 [h264 @ 0x5579c3358140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5579c3358140] error while decoding MB 0 43, bytestream 4030 [h264 @ 0x5579c2979b80] left block unavailable for requested intra mode [h264 @ 0x5579c2979b80] error while decoding MB 0 35, bytestream 12652 [h264 @ 0x5579c28d7b00] left block unavailable for requested intra4x4 ", "last_exit_at": "2026-07-01T08:30:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:30:38+09:00", "pid": 90106, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.488, "name": "ffmpeg", "pid": 90106, "rss_mb": 136.488, "state": "S (sleeping)", "threads": 106}, "restart_count": 491, "restart_count_per_hour": 22, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 492}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.562, "sample_count": 1900, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1400371, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:30:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.423, "last_frame_pushed_at": "2026-07-01T08:30:36+09:00", "last_frame_read_age_sec": 4.56, "last_frame_read_at": "2026-07-01T08:30:36+09:00", "mjpeg": {"age_sec": 4.406, "exists": true, "mtime": "2026-07-01T08:30:36.415529+09:00", "mtime_epoch": 1782862236.4155295, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172393}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:30:40+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5033.32, "name": "celery", "pid": 267, "rss_mb": 5033.32, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 50, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.667, "pushed": 1.667, "read": 1.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.255, "count": 50, "max_ms": 35.094, "p95_ms": 33.85}, "gstreamer_push": {"avg_ms": 1.071, "count": 50, "max_ms": 1.17, "p95_ms": 1.137}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.32, "count": 50, "max_ms": 41.042, "p95_ms": 35.062}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.877, "count": 50, "max_ms": 86.457, "p95_ms": 77.696}, "yolo11_640_fire_11class": {"avg_ms": 45.472, "count": 50, "max_ms": 62.899, "p95_ms": 58.275}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.749, "count": 50, "max_ms": 82.399, "p95_ms": 73.755}}, "truncated_count": 0}, "processing_total": {"avg_ms": 125.496, "count": 50, "max_ms": 157.045, "p95_ms": 146.389}, "read_interval": {"avg_ms": 511.565, "count": 50, "max_ms": 10132.189, "p95_ms": 169.039}, "triton_wait": {"avg_ms": 56.908, "count": 50, "max_ms": 86.47, "p95_ms": 77.707}, "visualize": {"avg_ms": 4.279, "count": 50, "max_ms": 5.097, "p95_ms": 4.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13161}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10811.449, "limit_mb": 8796093022207.996}, "detail": {"pid": 90213, "restart_count": 201, "start_count": 202}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 201, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:30:38+09:00", "last_error_summary": "[h264 @ 0x55af92ed6c80] error while decoding MB 47 39, bytestream -23 [in#0/rtsp @ 0x55af925d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af925d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:30:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:30:40+09:00", "pid": 90213, "process": {"exists": true, "fd_count": 5, "hwm_mb": 194.648, "name": "ffmpeg", "pid": 90213, "rss_mb": 194.648, "state": "S (sleeping)", "threads": 106}, "restart_count": 201, "restart_count_per_hour": 3, "rtsp_timeout_count": 328, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 202}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 192.312, "sample_count": 1928, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1579408, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:30:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.154, "last_frame_pushed_at": "2026-07-01T08:30:38+09:00", "last_frame_read_age_sec": 4.241, "last_frame_read_at": "2026-07-01T08:30:38+09:00", "mjpeg": {"age_sec": 4.137, "exists": true, "mtime": "2026-07-01T08:30:38.615566+09:00", "mtime_epoch": 1782862238.6155665, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 201112}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:30:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1805.109, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 24, "pushed_frames": 24, "read_frames": 24, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.8, "pushed": 0.8, "read": 0.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.9, "count": 24, "max_ms": 22.34, "p95_ms": 21.746}, "gstreamer_push": {"avg_ms": 1.214, "count": 24, "max_ms": 1.354, "p95_ms": 1.295}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.887, "count": 24, "max_ms": 57.455, "p95_ms": 48.927}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.501, "count": 24, "max_ms": 83.59, "p95_ms": 77.055}, "yolo11_640_fire_11class": {"avg_ms": 47.546, "count": 24, "max_ms": 60.073, "p95_ms": 59.295}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.008, "count": 24, "max_ms": 79.657, "p95_ms": 77.938}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.681, "count": 24, "max_ms": 111.351, "p95_ms": 105.119}, "read_interval": {"avg_ms": 1077.262, "count": 24, "max_ms": 13505.823, "p95_ms": 10167.74}, "triton_wait": {"avg_ms": 56.568, "count": 24, "max_ms": 83.6, "p95_ms": 77.95}, "visualize": {"avg_ms": 3.574, "count": 24, "max_ms": 6.094, "p95_ms": 4.613}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11679}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10811.215, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c671f75c40] error while decoding MB 37 31, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 447, "h264_decode_error_count": 1356, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:33:22+09:00", "last_error_summary": "[h264 @ 0x55c671f75c40] error while decoding MB 37 31, bytestream -7", "last_exit_at": "2026-07-01T08:30:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.253, "last_started_at": "2026-07-01T08:30:38+09:00", "pid": 90106, "process": {"exists": false, "pid": 90106}, "restart_count": 491, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 492}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.046, "sample_count": 1927, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1401481, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:33:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T08:33:22+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T08:33:22+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T08:33:22.030326+09:00", "mtime_epoch": 1782862402.0303257, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168011}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:33:22+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5033.836, "name": "celery", "pid": 267, "rss_mb": 5033.836, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.914, "pushed": 6.914, "read": 6.914}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.719, "count": 207, "max_ms": 50.082, "p95_ms": 37.644}, "gstreamer_push": {"avg_ms": 1.09, "count": 207, "max_ms": 1.483, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.363, "count": 207, "max_ms": 45.227, "p95_ms": 35.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.211, "count": 207, "max_ms": 84.515, "p95_ms": 81.397}, "yolo11_640_fire_11class": {"avg_ms": 48.049, "count": 207, "max_ms": 63.849, "p95_ms": 60.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.339, "count": 207, "max_ms": 84.501, "p95_ms": 78.812}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.868, "count": 207, "max_ms": 158.554, "p95_ms": 151.662}, "read_interval": {"avg_ms": 144.911, "count": 207, "max_ms": 169.564, "p95_ms": 164.396}, "triton_wait": {"avg_ms": 62.351, "count": 207, "max_ms": 84.527, "p95_ms": 81.409}, "visualize": {"avg_ms": 4.239, "count": 207, "max_ms": 6.109, "p95_ms": 4.656}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11679}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10810.785, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:33:22+09:00", "last_error_summary": "[h264 @ 0x55c671f75c40] error while decoding MB 37 31, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 447, "h264_decode_error_count": 1356, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:33:22+09:00", "last_error_summary": "[h264 @ 0x55c671f75c40] error while decoding MB 37 31, bytestream -7", "last_exit_at": "2026-07-01T08:33:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.336, "last_started_at": "2026-07-01T08:30:38+09:00", "pid": 90106, "process": {"exists": false, "pid": 90106}, "restart_count": 491, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 492}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.046, "sample_count": 1927, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1401482, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:33:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:33:22+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-01T08:33:22+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-01T08:33:22.030326+09:00", "mtime_epoch": 1782862402.0303257, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168011}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:33:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5033.836, "name": "celery", "pid": 267, "rss_mb": 5033.836, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.925, "pushed": 6.925, "read": 6.892}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.763, "count": 207, "max_ms": 50.082, "p95_ms": 37.644}, "gstreamer_push": {"avg_ms": 1.091, "count": 207, "max_ms": 1.483, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.374, "count": 207, "max_ms": 45.227, "p95_ms": 35.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.227, "count": 207, "max_ms": 84.515, "p95_ms": 81.397}, "yolo11_640_fire_11class": {"avg_ms": 48.062, "count": 207, "max_ms": 63.849, "p95_ms": 60.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.347, "count": 207, "max_ms": 84.501, "p95_ms": 78.812}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.959, "count": 207, "max_ms": 158.554, "p95_ms": 151.662}, "read_interval": {"avg_ms": 145.027, "count": 206, "max_ms": 169.564, "p95_ms": 164.396}, "triton_wait": {"avg_ms": 62.366, "count": 207, "max_ms": 84.527, "p95_ms": 81.409}, "visualize": {"avg_ms": 4.243, "count": 207, "max_ms": 6.109, "p95_ms": 4.656}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11679}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10810.742, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 447, "h264_decode_error_count": 1356, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:33:22+09:00", "last_error_summary": "[h264 @ 0x55c671f75c40] error while decoding MB 37 31, bytestream -7", "last_exit_at": "2026-07-01T08:33:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.341, "last_started_at": "2026-07-01T08:30:38+09:00", "pid": 90106, "process": {"exists": false, "pid": 90106}, "restart_count": 491, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 492}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.046, "sample_count": 1927, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1401482, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:33:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:33:22+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-01T08:33:22+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-01T08:33:22.030326+09:00", "mtime_epoch": 1782862402.0303257, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168011}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:33:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5033.836, "name": "celery", "pid": 267, "rss_mb": 5033.836, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.924, "pushed": 6.924, "read": 6.891}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.763, "count": 207, "max_ms": 50.082, "p95_ms": 37.644}, "gstreamer_push": {"avg_ms": 1.091, "count": 207, "max_ms": 1.483, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.374, "count": 207, "max_ms": 45.227, "p95_ms": 35.931}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.227, "count": 207, "max_ms": 84.515, "p95_ms": 81.397}, "yolo11_640_fire_11class": {"avg_ms": 48.062, "count": 207, "max_ms": 63.849, "p95_ms": 60.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.347, "count": 207, "max_ms": 84.501, "p95_ms": 78.812}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.959, "count": 207, "max_ms": 158.554, "p95_ms": 151.662}, "read_interval": {"avg_ms": 145.027, "count": 206, "max_ms": 169.564, "p95_ms": 164.396}, "triton_wait": {"avg_ms": 62.366, "count": 207, "max_ms": 84.527, "p95_ms": 81.409}, "visualize": {"avg_ms": 4.243, "count": 207, "max_ms": 6.109, "p95_ms": 4.656}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11679}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10919.359, "limit_mb": 8796093022207.996}, "detail": {"pid": 90365, "restart_count": 492, "start_count": 493}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 447, "h264_decode_error_count": 1356, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:33:22+09:00", "last_error_summary": "[h264 @ 0x55c671f75c40] error while decoding MB 37 31, bytestream -7", "last_exit_at": "2026-07-01T08:33:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:33:24+09:00", "pid": 90365, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.352, "name": "ffmpeg", "pid": 90365, "rss_mb": 139.352, "state": "S (sleeping)", "threads": 106}, "restart_count": 492, "restart_count_per_hour": 22, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 493}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.046, "sample_count": 1896, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1401482, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:33:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.334, "last_frame_pushed_at": "2026-07-01T08:33:22+09:00", "last_frame_read_age_sec": 4.465, "last_frame_read_at": "2026-07-01T08:33:22+09:00", "mjpeg": {"age_sec": 4.316, "exists": true, "mtime": "2026-07-01T08:33:22.170328+09:00", "mtime_epoch": 1782862402.1703281, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167806}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:33:26+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5033.836, "name": "celery", "pid": 267, "rss_mb": 5033.836, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 177, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.9, "pushed": 5.9, "read": 5.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.511, "count": 177, "max_ms": 40.281, "p95_ms": 35.724}, "gstreamer_push": {"avg_ms": 1.09, "count": 177, "max_ms": 1.483, "p95_ms": 1.157}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.185, "count": 177, "max_ms": 45.227, "p95_ms": 35.081}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.548, "count": 177, "max_ms": 84.515, "p95_ms": 81.586}, "yolo11_640_fire_11class": {"avg_ms": 48.214, "count": 177, "max_ms": 63.849, "p95_ms": 60.672}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.57, "count": 177, "max_ms": 84.501, "p95_ms": 79.272}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.998, "count": 177, "max_ms": 156.267, "p95_ms": 151.662}, "read_interval": {"avg_ms": 145.04, "count": 177, "max_ms": 169.564, "p95_ms": 164.324}, "triton_wait": {"avg_ms": 62.671, "count": 177, "max_ms": 84.527, "p95_ms": 81.598}, "visualize": {"avg_ms": 4.255, "count": 177, "max_ms": 6.109, "p95_ms": 4.549}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10813.031, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5598fc5c0000] error while decoding MB 44 33, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 448, "h264_decode_error_count": 1357, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:36:08+09:00", "last_error_summary": "[h264 @ 0x5598fc5c0000] error while decoding MB 44 33, bytestream -5", "last_exit_at": "2026-07-01T08:33:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.916, "last_started_at": "2026-07-01T08:33:24+09:00", "pid": 90365, "process": {"exists": false, "pid": 90365}, "restart_count": 492, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 493}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.102, "sample_count": 2023, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1402584, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:36:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T08:36:08+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T08:36:08+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T08:36:08.357143+09:00", "mtime_epoch": 1782862568.357143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:36:08+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5034.352, "name": "celery", "pid": 267, "rss_mb": 5034.352, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.848, "pushed": 6.848, "read": 6.848}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.324, "count": 205, "max_ms": 35.436, "p95_ms": 33.808}, "gstreamer_push": {"avg_ms": 1.081, "count": 205, "max_ms": 1.262, "p95_ms": 1.168}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.391, "count": 205, "max_ms": 41.624, "p95_ms": 36.444}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.219, "count": 205, "max_ms": 89.946, "p95_ms": 84.085}, "yolo11_640_fire_11class": {"avg_ms": 48.608, "count": 205, "max_ms": 65.229, "p95_ms": 61.948}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.193, "count": 205, "max_ms": 84.398, "p95_ms": 79.05}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.351, "count": 205, "max_ms": 162.456, "p95_ms": 155.01}, "read_interval": {"avg_ms": 146.4, "count": 205, "max_ms": 174.517, "p95_ms": 167.597}, "triton_wait": {"avg_ms": 64.5, "count": 205, "max_ms": 89.958, "p95_ms": 84.097}, "visualize": {"avg_ms": 4.267, "count": 205, "max_ms": 5.0, "p95_ms": 4.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10812.117, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:36:08+09:00", "last_error_summary": "[h264 @ 0x5598fc5c0000] error while decoding MB 44 33, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 448, "h264_decode_error_count": 1357, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:36:08+09:00", "last_error_summary": "[h264 @ 0x5598fc5c0000] error while decoding MB 44 33, bytestream -5", "last_exit_at": "2026-07-01T08:36:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.014, "last_started_at": "2026-07-01T08:33:24+09:00", "pid": 90365, "process": {"exists": false, "pid": 90365}, "restart_count": 492, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 493}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.102, "sample_count": 2024, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1402585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:36:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:36:08+09:00", "last_frame_read_age_sec": 0.149, "last_frame_read_at": "2026-07-01T08:36:08+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-01T08:36:08.357143+09:00", "mtime_epoch": 1782862568.357143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:36:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5034.352, "name": "celery", "pid": 267, "rss_mb": 5034.352, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.861, "pushed": 6.861, "read": 6.828}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.314, "count": 205, "max_ms": 35.436, "p95_ms": 33.808}, "gstreamer_push": {"avg_ms": 1.081, "count": 205, "max_ms": 1.262, "p95_ms": 1.168}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.394, "count": 205, "max_ms": 41.624, "p95_ms": 36.444}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.229, "count": 205, "max_ms": 89.946, "p95_ms": 84.085}, "yolo11_640_fire_11class": {"avg_ms": 48.643, "count": 205, "max_ms": 65.229, "p95_ms": 61.948}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.282, "count": 205, "max_ms": 84.398, "p95_ms": 79.05}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.424, "count": 205, "max_ms": 162.456, "p95_ms": 155.01}, "read_interval": {"avg_ms": 146.432, "count": 204, "max_ms": 174.517, "p95_ms": 167.597}, "triton_wait": {"avg_ms": 64.573, "count": 205, "max_ms": 89.958, "p95_ms": 84.097}, "visualize": {"avg_ms": 4.268, "count": 205, "max_ms": 5.0, "p95_ms": 4.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10812.109, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 448, "h264_decode_error_count": 1357, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:36:08+09:00", "last_error_summary": "[h264 @ 0x5598fc5c0000] error while decoding MB 44 33, bytestream -5", "last_exit_at": "2026-07-01T08:36:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.019, "last_started_at": "2026-07-01T08:33:24+09:00", "pid": 90365, "process": {"exists": false, "pid": 90365}, "restart_count": 492, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 493}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.102, "sample_count": 2024, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1402585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:36:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:36:08+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-01T08:36:08+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-01T08:36:08.357143+09:00", "mtime_epoch": 1782862568.357143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:36:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5034.352, "name": "celery", "pid": 267, "rss_mb": 5034.352, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.86, "pushed": 6.86, "read": 6.826}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.314, "count": 205, "max_ms": 35.436, "p95_ms": 33.808}, "gstreamer_push": {"avg_ms": 1.081, "count": 205, "max_ms": 1.262, "p95_ms": 1.168}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.394, "count": 205, "max_ms": 41.624, "p95_ms": 36.444}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.229, "count": 205, "max_ms": 89.946, "p95_ms": 84.085}, "yolo11_640_fire_11class": {"avg_ms": 48.643, "count": 205, "max_ms": 65.229, "p95_ms": 61.948}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.282, "count": 205, "max_ms": 84.398, "p95_ms": 79.05}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.424, "count": 205, "max_ms": 162.456, "p95_ms": 155.01}, "read_interval": {"avg_ms": 146.432, "count": 204, "max_ms": 174.517, "p95_ms": 167.597}, "triton_wait": {"avg_ms": 64.573, "count": 205, "max_ms": 89.958, "p95_ms": 84.097}, "visualize": {"avg_ms": 4.268, "count": 205, "max_ms": 5.0, "p95_ms": 4.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10920.586, "limit_mb": 8796093022207.996}, "detail": {"pid": 90472, "restart_count": 493, "start_count": 494}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 448, "h264_decode_error_count": 1357, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:36:08+09:00", "last_error_summary": "[h264 @ 0x5598fc5c0000] error while decoding MB 44 33, bytestream -5", "last_exit_at": "2026-07-01T08:36:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:36:11+09:00", "pid": 90472, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.434, "name": "ffmpeg", "pid": 90472, "rss_mb": 139.434, "state": "S (sleeping)", "threads": 106}, "restart_count": 493, "restart_count_per_hour": 22, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 494}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.102, "sample_count": 1992, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1402585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:36:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.612, "last_frame_pushed_at": "2026-07-01T08:36:08+09:00", "last_frame_read_age_sec": 4.756, "last_frame_read_at": "2026-07-01T08:36:08+09:00", "mjpeg": {"age_sec": 4.597, "exists": true, "mtime": "2026-07-01T08:36:08.509145+09:00", "mtime_epoch": 1782862568.5091455, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:36:13+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5034.352, "name": "celery", "pid": 267, "rss_mb": 5034.352, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.842, "pushed": 5.842, "read": 5.809}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.32, "count": 175, "max_ms": 34.973, "p95_ms": 33.72}, "gstreamer_push": {"avg_ms": 1.082, "count": 175, "max_ms": 1.262, "p95_ms": 1.169}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.534, "count": 175, "max_ms": 41.624, "p95_ms": 36.597}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.411, "count": 175, "max_ms": 89.946, "p95_ms": 82.254}, "yolo11_640_fire_11class": {"avg_ms": 48.552, "count": 175, "max_ms": 65.229, "p95_ms": 61.948}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.597, "count": 175, "max_ms": 84.398, "p95_ms": 79.562}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.557, "count": 175, "max_ms": 161.547, "p95_ms": 152.793}, "read_interval": {"avg_ms": 145.539, "count": 174, "max_ms": 173.296, "p95_ms": 164.829}, "triton_wait": {"avg_ms": 63.808, "count": 175, "max_ms": 89.958, "p95_ms": 82.264}, "visualize": {"avg_ms": 4.265, "count": 175, "max_ms": 5.0, "p95_ms": 4.5}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11697}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10815.398, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5616500919c0] error while decoding MB 4 28, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 449, "h264_decode_error_count": 1358, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:38:54+09:00", "last_error_summary": "[h264 @ 0x5616500919c0] error while decoding MB 4 28, bytestream -5", "last_exit_at": "2026-07-01T08:36:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.541, "last_started_at": "2026-07-01T08:36:11+09:00", "pid": 90472, "process": {"exists": false, "pid": 90472}, "restart_count": 493, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 494}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.042, "sample_count": 2012, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1403686, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:38:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T08:38:54+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T08:38:54+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T08:38:54.599965+09:00", "mtime_epoch": 1782862734.5999653, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:38:54+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5034.867, "name": "celery", "pid": 267, "rss_mb": 5034.867, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.911, "pushed": 6.911, "read": 6.911}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.3, "count": 207, "max_ms": 39.932, "p95_ms": 34.464}, "gstreamer_push": {"avg_ms": 1.089, "count": 207, "max_ms": 1.874, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.136, "count": 207, "max_ms": 41.697, "p95_ms": 37.411}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.882, "count": 207, "max_ms": 89.165, "p95_ms": 82.782}, "yolo11_640_fire_11class": {"avg_ms": 47.436, "count": 207, "max_ms": 65.213, "p95_ms": 60.083}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.334, "count": 207, "max_ms": 89.177, "p95_ms": 80.496}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.976, "count": 207, "max_ms": 159.009, "p95_ms": 152.092}, "read_interval": {"avg_ms": 145.039, "count": 207, "max_ms": 173.052, "p95_ms": 164.402}, "triton_wait": {"avg_ms": 63.108, "count": 207, "max_ms": 89.187, "p95_ms": 82.794}, "visualize": {"avg_ms": 4.272, "count": 207, "max_ms": 5.579, "p95_ms": 4.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11697}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10815.051, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:38:54+09:00", "last_error_summary": "[h264 @ 0x5616500919c0] error while decoding MB 4 28, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 449, "h264_decode_error_count": 1358, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:38:54+09:00", "last_error_summary": "[h264 @ 0x5616500919c0] error while decoding MB 4 28, bytestream -5", "last_exit_at": "2026-07-01T08:38:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.625, "last_started_at": "2026-07-01T08:36:11+09:00", "pid": 90472, "process": {"exists": false, "pid": 90472}, "restart_count": 493, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 494}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.042, "sample_count": 2012, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1403687, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:38:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:38:54+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-01T08:38:54+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-01T08:38:54.599965+09:00", "mtime_epoch": 1782862734.5999653, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:38:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5034.867, "name": "celery", "pid": 267, "rss_mb": 5034.867, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.924, "pushed": 6.924, "read": 6.89}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.317, "count": 207, "max_ms": 39.932, "p95_ms": 34.464}, "gstreamer_push": {"avg_ms": 1.089, "count": 207, "max_ms": 1.874, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.125, "count": 207, "max_ms": 41.697, "p95_ms": 37.411}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.901, "count": 207, "max_ms": 89.165, "p95_ms": 82.782}, "yolo11_640_fire_11class": {"avg_ms": 47.435, "count": 207, "max_ms": 65.213, "p95_ms": 60.083}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.331, "count": 207, "max_ms": 89.177, "p95_ms": 80.496}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.014, "count": 207, "max_ms": 159.009, "p95_ms": 152.092}, "read_interval": {"avg_ms": 145.094, "count": 206, "max_ms": 173.052, "p95_ms": 164.402}, "triton_wait": {"avg_ms": 63.127, "count": 207, "max_ms": 89.187, "p95_ms": 82.794}, "visualize": {"avg_ms": 4.28, "count": 207, "max_ms": 5.792, "p95_ms": 4.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11697}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10815.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 449, "h264_decode_error_count": 1358, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:38:54+09:00", "last_error_summary": "[h264 @ 0x5616500919c0] error while decoding MB 4 28, bytestream -5", "last_exit_at": "2026-07-01T08:38:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.631, "last_started_at": "2026-07-01T08:36:11+09:00", "pid": 90472, "process": {"exists": false, "pid": 90472}, "restart_count": 493, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 494}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.042, "sample_count": 2012, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1403687, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:38:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:38:54+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-01T08:38:54+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-01T08:38:54.599965+09:00", "mtime_epoch": 1782862734.5999653, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:38:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5034.867, "name": "celery", "pid": 267, "rss_mb": 5034.867, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.923, "pushed": 6.923, "read": 6.889}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.317, "count": 207, "max_ms": 39.932, "p95_ms": 34.464}, "gstreamer_push": {"avg_ms": 1.089, "count": 207, "max_ms": 1.874, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.125, "count": 207, "max_ms": 41.697, "p95_ms": 37.411}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.901, "count": 207, "max_ms": 89.165, "p95_ms": 82.782}, "yolo11_640_fire_11class": {"avg_ms": 47.435, "count": 207, "max_ms": 65.213, "p95_ms": 60.083}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.331, "count": 207, "max_ms": 89.177, "p95_ms": 80.496}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.014, "count": 207, "max_ms": 159.009, "p95_ms": 152.092}, "read_interval": {"avg_ms": 145.094, "count": 206, "max_ms": 173.052, "p95_ms": 164.402}, "triton_wait": {"avg_ms": 63.127, "count": 207, "max_ms": 89.187, "p95_ms": 82.794}, "visualize": {"avg_ms": 4.28, "count": 207, "max_ms": 5.792, "p95_ms": 4.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11697}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10920.23, "limit_mb": 8796093022207.996}, "detail": {"pid": 90579, "restart_count": 494, "start_count": 495}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 449, "h264_decode_error_count": 1358, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:38:54+09:00", "last_error_summary": "[h264 @ 0x5616500919c0] error while decoding MB 4 28, bytestream -5", "last_exit_at": "2026-07-01T08:38:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:38:57+09:00", "pid": 90579, "process": {"exists": true, "fd_count": 5, "hwm_mb": 135.836, "name": "ffmpeg", "pid": 90579, "rss_mb": 135.836, "state": "S (sleeping)", "threads": 106}, "restart_count": 494, "restart_count_per_hour": 22, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 495}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.042, "sample_count": 1979, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1403687, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:38:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.919, "last_frame_pushed_at": "2026-07-01T08:38:54+09:00", "last_frame_read_age_sec": 5.051, "last_frame_read_at": "2026-07-01T08:38:54+09:00", "mjpeg": {"age_sec": 4.905, "exists": true, "mtime": "2026-07-01T08:38:54.739968+09:00", "mtime_epoch": 1782862734.7399678, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166931}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:38:59+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5034.867, "name": "celery", "pid": 267, "rss_mb": 5034.867, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.806, "pushed": 5.806, "read": 5.772}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.221, "count": 173, "max_ms": 39.932, "p95_ms": 34.778}, "gstreamer_push": {"avg_ms": 1.086, "count": 174, "max_ms": 1.874, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.084, "count": 173, "max_ms": 41.697, "p95_ms": 37.657}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.85, "count": 173, "max_ms": 89.165, "p95_ms": 83.098}, "yolo11_640_fire_11class": {"avg_ms": 47.187, "count": 173, "max_ms": 64.097, "p95_ms": 60.01}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.997, "count": 173, "max_ms": 89.177, "p95_ms": 81.033}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.757, "count": 174, "max_ms": 159.009, "p95_ms": 152.092}, "read_interval": {"avg_ms": 144.784, "count": 173, "max_ms": 173.052, "p95_ms": 164.253}, "triton_wait": {"avg_ms": 63.006, "count": 173, "max_ms": 89.187, "p95_ms": 83.108}, "visualize": {"avg_ms": 4.286, "count": 173, "max_ms": 5.792, "p95_ms": 4.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13210}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10756.484, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5636b95b3000] error while decoding MB 77 38, bytestream -13 [in#0/rtsp @ 0x5636b956be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5636b956be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 202, "h264_decode_error_count": 208, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:41:30+09:00", "last_error_summary": "[h264 @ 0x5636b95b3000] error while decoding MB 77 38, bytestream -13 [in#0/rtsp @ 0x5636b956be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5636b956be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:30:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 649.596, "last_started_at": "2026-07-01T08:30:40+09:00", "pid": 90213, "process": {"exists": false, "pid": 90213}, "restart_count": 201, "restart_count_per_hour": 3, "rtsp_timeout_count": 330, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 202}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.367, "sample_count": 2503, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1585236, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:41:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.084, "last_frame_pushed_at": "2026-07-01T08:41:30+09:00", "last_frame_read_age_sec": 0.075, "last_frame_read_at": "2026-07-01T08:41:30+09:00", "mjpeg": {"age_sec": 0.068, "exists": true, "mtime": "2026-07-01T08:41:30.280221+09:00", "mtime_epoch": 1782862890.2802205, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218062}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:41:30+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1807.879, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 92, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.078, "pushed": 3.078, "read": 3.078}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.261, "count": 92, "max_ms": 27.835, "p95_ms": 24.868}, "gstreamer_push": {"avg_ms": 1.094, "count": 92, "max_ms": 1.362, "p95_ms": 1.287}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.31, "count": 92, "max_ms": 59.769, "p95_ms": 53.718}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.774, "count": 92, "max_ms": 85.154, "p95_ms": 83.823}, "yolo11_640_fire_11class": {"avg_ms": 52.716, "count": 92, "max_ms": 75.159, "p95_ms": 73.344}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.585, "count": 92, "max_ms": 83.4, "p95_ms": 79.66}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.558, "count": 92, "max_ms": 175.353, "p95_ms": 148.277}, "read_interval": {"avg_ms": 325.575, "count": 92, "max_ms": 10159.933, "p95_ms": 161.586}, "triton_wait": {"avg_ms": 63.072, "count": 92, "max_ms": 85.166, "p95_ms": 83.834}, "visualize": {"avg_ms": 4.043, "count": 92, "max_ms": 5.715, "p95_ms": 5.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.886, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13210}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10756.199, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:41:30+09:00", "last_error_summary": "[h264 @ 0x5636b95b3000] error while decoding MB 77 38, bytestream -13 [in#0/rtsp @ 0x5636b956be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5636b956be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 202, "h264_decode_error_count": 208, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:41:30+09:00", "last_error_summary": "[h264 @ 0x5636b95b3000] error while decoding MB 77 38, bytestream -13 [in#0/rtsp @ 0x5636b956be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5636b956be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:41:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 649.659, "last_started_at": "2026-07-01T08:30:40+09:00", "pid": 90213, "process": {"exists": false, "pid": 90213}, "restart_count": 201, "restart_count_per_hour": 3, "rtsp_timeout_count": 330, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 202}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.367, "sample_count": 2504, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1585237, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:41:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:41:30+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-01T08:41:30+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-01T08:41:30.280221+09:00", "mtime_epoch": 1782862890.2802205, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218062}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:41:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1807.879, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.105, "pushed": 3.105, "read": 3.072}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.312, "count": 93, "max_ms": 27.835, "p95_ms": 24.871}, "gstreamer_push": {"avg_ms": 1.093, "count": 93, "max_ms": 1.362, "p95_ms": 1.287}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.374, "count": 93, "max_ms": 59.769, "p95_ms": 53.718}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.711, "count": 93, "max_ms": 85.154, "p95_ms": 83.823}, "yolo11_640_fire_11class": {"avg_ms": 52.673, "count": 93, "max_ms": 75.159, "p95_ms": 73.344}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.549, "count": 93, "max_ms": 83.4, "p95_ms": 79.66}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.777, "count": 93, "max_ms": 175.353, "p95_ms": 148.277}, "read_interval": {"avg_ms": 325.575, "count": 92, "max_ms": 10159.933, "p95_ms": 161.586}, "triton_wait": {"avg_ms": 63.007, "count": 93, "max_ms": 85.166, "p95_ms": 83.834}, "visualize": {"avg_ms": 4.081, "count": 93, "max_ms": 7.603, "p95_ms": 5.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13210}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10756.191, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 202, "h264_decode_error_count": 208, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:41:30+09:00", "last_error_summary": "[h264 @ 0x5636b95b3000] error while decoding MB 77 38, bytestream -13 [in#0/rtsp @ 0x5636b956be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5636b956be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:41:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 649.664, "last_started_at": "2026-07-01T08:30:40+09:00", "pid": 90213, "process": {"exists": false, "pid": 90213}, "restart_count": 201, "restart_count_per_hour": 3, "rtsp_timeout_count": 330, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 202}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.367, "sample_count": 2504, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1585237, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:41:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:41:30+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-01T08:41:30+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-01T08:41:30.280221+09:00", "mtime_epoch": 1782862890.2802205, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218062}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:41:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1807.879, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.105, "pushed": 3.105, "read": 3.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.312, "count": 93, "max_ms": 27.835, "p95_ms": 24.871}, "gstreamer_push": {"avg_ms": 1.093, "count": 93, "max_ms": 1.362, "p95_ms": 1.287}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.374, "count": 93, "max_ms": 59.769, "p95_ms": 53.718}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.711, "count": 93, "max_ms": 85.154, "p95_ms": 83.823}, "yolo11_640_fire_11class": {"avg_ms": 52.673, "count": 93, "max_ms": 75.159, "p95_ms": 73.344}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.549, "count": 93, "max_ms": 83.4, "p95_ms": 79.66}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.777, "count": 93, "max_ms": 175.353, "p95_ms": 148.277}, "read_interval": {"avg_ms": 325.575, "count": 92, "max_ms": 10159.933, "p95_ms": 161.586}, "triton_wait": {"avg_ms": 63.007, "count": 93, "max_ms": 85.166, "p95_ms": 83.834}, "visualize": {"avg_ms": 4.081, "count": 93, "max_ms": 7.603, "p95_ms": 5.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13210}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10880.508, "limit_mb": 8796093022207.996}, "detail": {"pid": 90686, "restart_count": 202, "start_count": 203}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 202, "h264_decode_error_count": 208, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:41:30+09:00", "last_error_summary": "[h264 @ 0x5636b95b3000] error while decoding MB 77 38, bytestream -13 [in#0/rtsp @ 0x5636b956be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5636b956be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:41:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:41:33+09:00", "pid": 90686, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.941, "name": "ffmpeg", "pid": 90686, "rss_mb": 156.941, "state": "R (running)", "threads": 49}, "restart_count": 202, "restart_count_per_hour": 4, "rtsp_timeout_count": 330, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 203}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.543, "sample_count": 2455, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1585237, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:41:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.877, "last_frame_pushed_at": "2026-07-01T08:41:30+09:00", "last_frame_read_age_sec": 5.011, "last_frame_read_at": "2026-07-01T08:41:30+09:00", "mjpeg": {"age_sec": 4.864, "exists": true, "mtime": "2026-07-01T08:41:30.420225+09:00", "mtime_epoch": 1782862890.4202247, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217778}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:41:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1807.879, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 51, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.738, "pushed": 1.738, "read": 1.705}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.977, "count": 52, "max_ms": 27.835, "p95_ms": 26.01}, "gstreamer_push": {"avg_ms": 1.043, "count": 52, "max_ms": 1.362, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.551, "count": 52, "max_ms": 59.417, "p95_ms": 53.598}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.401, "count": 52, "max_ms": 84.847, "p95_ms": 78.689}, "yolo11_640_fire_11class": {"avg_ms": 48.561, "count": 52, "max_ms": 61.615, "p95_ms": 56.566}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.444, "count": 52, "max_ms": 83.4, "p95_ms": 75.896}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.507, "count": 52, "max_ms": 175.353, "p95_ms": 150.125}, "read_interval": {"avg_ms": 490.03, "count": 51, "max_ms": 10159.933, "p95_ms": 185.807}, "triton_wait": {"avg_ms": 58.597, "count": 52, "max_ms": 84.858, "p95_ms": 78.699}, "visualize": {"avg_ms": 4.306, "count": 52, "max_ms": 7.603, "p95_ms": 5.42}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10790.941, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5603e47a1540] error while decoding MB 64 40, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 450, "h264_decode_error_count": 1359, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:41:41+09:00", "last_error_summary": "[h264 @ 0x5603e47a1540] error while decoding MB 64 40, bytestream -5", "last_exit_at": "2026-07-01T08:38:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.537, "last_started_at": "2026-07-01T08:38:57+09:00", "pid": 90579, "process": {"exists": false, "pid": 90579}, "restart_count": 494, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 495}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.296, "sample_count": 2027, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1404804, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:41:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.05, "last_frame_pushed_at": "2026-07-01T08:41:41+09:00", "last_frame_read_age_sec": 0.04, "last_frame_read_at": "2026-07-01T08:41:41+09:00", "mjpeg": {"age_sec": 0.033, "exists": true, "mtime": "2026-07-01T08:41:41.148543+09:00", "mtime_epoch": 1782862901.1485426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169702}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:41:41+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5035.383, "name": "celery", "pid": 267, "rss_mb": 5035.383, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.453, "pushed": 7.453, "read": 7.453}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget"], "latency_ms": {"analyze": {"avg_ms": 30.049, "count": 223, "max_ms": 42.526, "p95_ms": 33.269}, "gstreamer_push": {"avg_ms": 1.035, "count": 223, "max_ms": 1.155, "p95_ms": 1.115}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.816, "count": 223, "max_ms": 40.834, "p95_ms": 34.116}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.219, "count": 223, "max_ms": 89.885, "p95_ms": 57.761}, "yolo11_640_fire_11class": {"avg_ms": 43.303, "count": 223, "max_ms": 57.125, "p95_ms": 45.685}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.69, "count": 223, "max_ms": 89.068, "p95_ms": 52.367}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.691, "count": 223, "max_ms": 160.784, "p95_ms": 129.844}, "read_interval": {"avg_ms": 134.55, "count": 223, "max_ms": 173.201, "p95_ms": 141.682}, "triton_wait": {"avg_ms": 54.235, "count": 223, "max_ms": 89.898, "p95_ms": 57.772}, "visualize": {"avg_ms": 4.273, "count": 223, "max_ms": 5.403, "p95_ms": 4.508}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10792.805, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:41:41+09:00", "last_error_summary": "[h264 @ 0x5603e47a1540] error while decoding MB 64 40, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 450, "h264_decode_error_count": 1359, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:41:41+09:00", "last_error_summary": "[h264 @ 0x5603e47a1540] error while decoding MB 64 40, bytestream -5", "last_exit_at": "2026-07-01T08:41:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.635, "last_started_at": "2026-07-01T08:38:57+09:00", "pid": 90579, "process": {"exists": false, "pid": 90579}, "restart_count": 494, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 495}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.296, "sample_count": 2027, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1404805, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:41:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:41:41+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-01T08:41:41+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-01T08:41:41.148543+09:00", "mtime_epoch": 1782862901.1485426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169702}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:41:41+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5035.383, "name": "celery", "pid": 267, "rss_mb": 5035.383, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.461, "pushed": 7.461, "read": 7.428}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget"], "latency_ms": {"analyze": {"avg_ms": 30.076, "count": 223, "max_ms": 42.526, "p95_ms": 33.676}, "gstreamer_push": {"avg_ms": 1.035, "count": 223, "max_ms": 1.155, "p95_ms": 1.115}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.814, "count": 223, "max_ms": 40.834, "p95_ms": 34.116}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.24, "count": 223, "max_ms": 89.885, "p95_ms": 57.784}, "yolo11_640_fire_11class": {"avg_ms": 43.322, "count": 223, "max_ms": 57.125, "p95_ms": 45.84}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.707, "count": 223, "max_ms": 89.068, "p95_ms": 52.783}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.732, "count": 223, "max_ms": 160.784, "p95_ms": 129.916}, "read_interval": {"avg_ms": 134.549, "count": 222, "max_ms": 173.201, "p95_ms": 141.682}, "triton_wait": {"avg_ms": 54.256, "count": 223, "max_ms": 89.898, "p95_ms": 57.796}, "visualize": {"avg_ms": 4.273, "count": 223, "max_ms": 5.403, "p95_ms": 4.508}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10792.773, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 450, "h264_decode_error_count": 1359, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:41:41+09:00", "last_error_summary": "[h264 @ 0x5603e47a1540] error while decoding MB 64 40, bytestream -5", "last_exit_at": "2026-07-01T08:41:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.64, "last_started_at": "2026-07-01T08:38:57+09:00", "pid": 90579, "process": {"exists": false, "pid": 90579}, "restart_count": 494, "restart_count_per_hour": 21, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 495}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.296, "sample_count": 2027, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1404805, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:41:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T08:41:41+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-01T08:41:41+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-01T08:41:41.148543+09:00", "mtime_epoch": 1782862901.1485426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169702}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:41:41+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5035.383, "name": "celery", "pid": 267, "rss_mb": 5035.383, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.46, "pushed": 7.46, "read": 7.427}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget"], "latency_ms": {"analyze": {"avg_ms": 30.076, "count": 223, "max_ms": 42.526, "p95_ms": 33.676}, "gstreamer_push": {"avg_ms": 1.035, "count": 223, "max_ms": 1.155, "p95_ms": 1.115}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.814, "count": 223, "max_ms": 40.834, "p95_ms": 34.116}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.24, "count": 223, "max_ms": 89.885, "p95_ms": 57.784}, "yolo11_640_fire_11class": {"avg_ms": 43.322, "count": 223, "max_ms": 57.125, "p95_ms": 45.84}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.707, "count": 223, "max_ms": 89.068, "p95_ms": 52.783}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.732, "count": 223, "max_ms": 160.784, "p95_ms": 129.916}, "read_interval": {"avg_ms": 134.549, "count": 222, "max_ms": 173.201, "p95_ms": 141.682}, "triton_wait": {"avg_ms": 54.256, "count": 223, "max_ms": 89.898, "p95_ms": 57.796}, "visualize": {"avg_ms": 4.273, "count": 223, "max_ms": 5.403, "p95_ms": 4.508}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11706}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10827.059, "limit_mb": 8796093022207.996}, "detail": {"pid": 90838, "restart_count": 495, "start_count": 496}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 450, "h264_decode_error_count": 1359, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:41:41+09:00", "last_error_summary": "[h264 @ 0x5603e47a1540] error while decoding MB 64 40, bytestream -5", "last_exit_at": "2026-07-01T08:41:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:41:44+09:00", "pid": 90838, "process": {"exists": true, "fd_count": 4, "hwm_mb": 49.941, "name": "ffmpeg", "pid": 90838, "rss_mb": 49.941, "state": "S (sleeping)", "threads": 1}, "restart_count": 495, "restart_count_per_hour": 22, "rtsp_timeout_count": 394, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 496}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.296, "sample_count": 1995, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1404805, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:41:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.822, "last_frame_pushed_at": "2026-07-01T08:41:41+09:00", "last_frame_read_age_sec": 4.956, "last_frame_read_at": "2026-07-01T08:41:41+09:00", "mjpeg": {"age_sec": 4.808, "exists": true, "mtime": "2026-07-01T08:41:41.288547+09:00", "mtime_epoch": 1782862901.2885468, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:41:46+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5035.383, "name": "celery", "pid": 267, "rss_mb": 5035.383, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 187, "pushed_frames": 187, "read_frames": 186, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.257, "pushed": 6.257, "read": 6.223}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget"], "latency_ms": {"analyze": {"avg_ms": 30.214, "count": 187, "max_ms": 42.526, "p95_ms": 34.158}, "gstreamer_push": {"avg_ms": 1.034, "count": 187, "max_ms": 1.155, "p95_ms": 1.121}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.798, "count": 187, "max_ms": 40.834, "p95_ms": 34.124}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.378, "count": 187, "max_ms": 89.885, "p95_ms": 57.761}, "yolo11_640_fire_11class": {"avg_ms": 43.417, "count": 187, "max_ms": 57.125, "p95_ms": 46.278}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.902, "count": 187, "max_ms": 89.068, "p95_ms": 52.88}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.866, "count": 187, "max_ms": 160.784, "p95_ms": 130.715}, "read_interval": {"avg_ms": 134.697, "count": 186, "max_ms": 173.201, "p95_ms": 142.253}, "triton_wait": {"avg_ms": 54.394, "count": 187, "max_ms": 89.898, "p95_ms": 57.772}, "visualize": {"avg_ms": 4.267, "count": 187, "max_ms": 5.403, "p95_ms": 4.508}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11715}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10817.738, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5558455dafc0] cabac decode of qscale diff failed at 16 32 [h264 @ 0x5558455dafc0] error while decoding MB 16 32, bytestream 746 [h264 @ 0x5558460c5740] left block unavailable for requested intra mode [h264 @ 0x5558460c5740] error while decoding MB 0 18, bytestream 6694 [h264 @ 0x5558461945c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5558461945c0] error while decoding MB 0 31, bytestream 5266 [in#0/rtsp @ 0x555845555e00] Error during demuxing: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 451, "h264_decode_error_count": 1363, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:44:36+09:00", "last_error_summary": "[h264 @ 0x5558455dafc0] cabac decode of qscale diff failed at 16 32 [h264 @ 0x5558455dafc0] error while decoding MB 16 32, bytestream 746 [h264 @ 0x5558460c5740] left block unavailable for requested intra mode [h264 @ 0x5558460c5740] error while decoding MB 0 18, bytestream 6694 [h264 @ 0x5558461945c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5558461945c0] error while decoding MB 0 31, bytestream 5266 [in#0/rtsp @ 0x555845555e00] Error during demuxing: Connection timed out", "last_exit_at": "2026-07-01T08:41:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 172.44, "last_started_at": "2026-07-01T08:41:44+09:00", "pid": 90838, "process": {"exists": false, "pid": 90838}, "restart_count": 495, "restart_count_per_hour": 21, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 496}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.182, "sample_count": 1906, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1405839, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:44:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T08:44:36+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T08:44:36+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T08:44:36.489430+09:00", "mtime_epoch": 1782863076.4894304, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168916}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:44:36+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5035.898, "name": "celery", "pid": 267, "rss_mb": 5035.898, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 75, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.507, "pushed": 2.507, "read": 2.507}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.652, "count": 75, "max_ms": 34.569, "p95_ms": 32.982}, "gstreamer_push": {"avg_ms": 1.059, "count": 75, "max_ms": 1.183, "p95_ms": 1.137}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.87, "count": 75, "max_ms": 40.66, "p95_ms": 40.198}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.257, "count": 75, "max_ms": 87.655, "p95_ms": 82.436}, "yolo11_640_fire_11class": {"avg_ms": 50.347, "count": 75, "max_ms": 63.441, "p95_ms": 57.276}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.079, "count": 75, "max_ms": 81.833, "p95_ms": 76.616}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.794, "count": 75, "max_ms": 158.288, "p95_ms": 152.983}, "read_interval": {"avg_ms": 399.946, "count": 75, "max_ms": 18765.028, "p95_ms": 164.874}, "triton_wait": {"avg_ms": 70.269, "count": 75, "max_ms": 87.668, "p95_ms": 82.448}, "visualize": {"avg_ms": 4.269, "count": 75, "max_ms": 4.816, "p95_ms": 4.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11715}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10817.324, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:44:36+09:00", "last_error_summary": "[h264 @ 0x5558455dafc0] cabac decode of qscale diff failed at 16 32 [h264 @ 0x5558455dafc0] error while decoding MB 16 32, bytestream 746 [h264 @ 0x5558460c5740] left block unavailable for requested intra mode [h264 @ 0x5558460c5740] error while decoding MB 0 18, bytestream 6694 [h264 @ 0x5558461945c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5558461945c0] error while decoding MB 0 31, bytestream 5266 [in#0/rtsp @ 0x555845555e00] Error during demuxing: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 451, "h264_decode_error_count": 1363, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:44:36+09:00", "last_error_summary": "[h264 @ 0x5558455dafc0] cabac decode of qscale diff failed at 16 32 [h264 @ 0x5558455dafc0] error while decoding MB 16 32, bytestream 746 [h264 @ 0x5558460c5740] left block unavailable for requested intra mode [h264 @ 0x5558460c5740] error while decoding MB 0 18, bytestream 6694 [h264 @ 0x5558461945c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5558461945c0] error while decoding MB 0 31, bytestream 5266 [in#0/rtsp @ 0x555845555e00] Error during demuxing: Connection timed out", "last_exit_at": "2026-07-01T08:44:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 172.557, "last_started_at": "2026-07-01T08:41:44+09:00", "pid": 90838, "process": {"exists": false, "pid": 90838}, "restart_count": 495, "restart_count_per_hour": 21, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 496}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.215, "sample_count": 1906, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1405840, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:44:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:44:36+09:00", "last_frame_read_age_sec": 0.169, "last_frame_read_at": "2026-07-01T08:44:36+09:00", "mjpeg": {"age_sec": 0.163, "exists": true, "mtime": "2026-07-01T08:44:36.489430+09:00", "mtime_epoch": 1782863076.4894304, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168916}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:44:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5035.898, "name": "celery", "pid": 267, "rss_mb": 5035.898, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 74, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.508, "pushed": 2.508, "read": 2.475}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.7, "count": 75, "max_ms": 34.569, "p95_ms": 32.982}, "gstreamer_push": {"avg_ms": 1.057, "count": 75, "max_ms": 1.183, "p95_ms": 1.137}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.0, "count": 75, "max_ms": 41.453, "p95_ms": 40.35}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.376, "count": 75, "max_ms": 87.655, "p95_ms": 82.436}, "yolo11_640_fire_11class": {"avg_ms": 50.295, "count": 75, "max_ms": 63.441, "p95_ms": 57.276}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.093, "count": 75, "max_ms": 81.833, "p95_ms": 76.616}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.242, "count": 75, "max_ms": 162.103, "p95_ms": 153.147}, "read_interval": {"avg_ms": 403.461, "count": 74, "max_ms": 18765.028, "p95_ms": 164.874}, "triton_wait": {"avg_ms": 70.388, "count": 75, "max_ms": 87.668, "p95_ms": 82.448}, "visualize": {"avg_ms": 4.28, "count": 75, "max_ms": 5.076, "p95_ms": 4.69}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11715}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10817.316, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 451, "h264_decode_error_count": 1363, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:44:36+09:00", "last_error_summary": "[h264 @ 0x5558455dafc0] cabac decode of qscale diff failed at 16 32 [h264 @ 0x5558455dafc0] error while decoding MB 16 32, bytestream 746 [h264 @ 0x5558460c5740] left block unavailable for requested intra mode [h264 @ 0x5558460c5740] error while decoding MB 0 18, bytestream 6694 [h264 @ 0x5558461945c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5558461945c0] error while decoding MB 0 31, bytestream 5266 [in#0/rtsp @ 0x555845555e00] Error during demuxing: Connection timed out", "last_exit_at": "2026-07-01T08:44:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 172.561, "last_started_at": "2026-07-01T08:41:44+09:00", "pid": 90838, "process": {"exists": false, "pid": 90838}, "restart_count": 495, "restart_count_per_hour": 21, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 496}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.215, "sample_count": 1906, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1405840, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:44:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T08:44:36+09:00", "last_frame_read_age_sec": 0.172, "last_frame_read_at": "2026-07-01T08:44:36+09:00", "mjpeg": {"age_sec": 0.167, "exists": true, "mtime": "2026-07-01T08:44:36.489430+09:00", "mtime_epoch": 1782863076.4894304, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168916}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:44:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5035.898, "name": "celery", "pid": 267, "rss_mb": 5035.898, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 74, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.508, "pushed": 2.508, "read": 2.474}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.7, "count": 75, "max_ms": 34.569, "p95_ms": 32.982}, "gstreamer_push": {"avg_ms": 1.057, "count": 75, "max_ms": 1.183, "p95_ms": 1.137}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.0, "count": 75, "max_ms": 41.453, "p95_ms": 40.35}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.376, "count": 75, "max_ms": 87.655, "p95_ms": 82.436}, "yolo11_640_fire_11class": {"avg_ms": 50.295, "count": 75, "max_ms": 63.441, "p95_ms": 57.276}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.093, "count": 75, "max_ms": 81.833, "p95_ms": 76.616}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.242, "count": 75, "max_ms": 162.103, "p95_ms": 153.147}, "read_interval": {"avg_ms": 403.461, "count": 74, "max_ms": 18765.028, "p95_ms": 164.874}, "triton_wait": {"avg_ms": 70.388, "count": 75, "max_ms": 87.668, "p95_ms": 82.448}, "visualize": {"avg_ms": 4.28, "count": 75, "max_ms": 5.076, "p95_ms": 4.69}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11715}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10929.766, "limit_mb": 8796093022207.996}, "detail": {"pid": 90945, "restart_count": 496, "start_count": 497}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 451, "h264_decode_error_count": 1363, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:44:36+09:00", "last_error_summary": "[h264 @ 0x5558455dafc0] cabac decode of qscale diff failed at 16 32 [h264 @ 0x5558455dafc0] error while decoding MB 16 32, bytestream 746 [h264 @ 0x5558460c5740] left block unavailable for requested intra mode [h264 @ 0x5558460c5740] error while decoding MB 0 18, bytestream 6694 [h264 @ 0x5558461945c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5558461945c0] error while decoding MB 0 31, bytestream 5266 [in#0/rtsp @ 0x555845555e00] Error during demuxing: Connection timed out", "last_exit_at": "2026-07-01T08:44:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:44:39+09:00", "pid": 90945, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.133, "name": "ffmpeg", "pid": 90945, "rss_mb": 142.133, "state": "S (sleeping)", "threads": 106}, "restart_count": 496, "restart_count_per_hour": 22, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 497}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.215, "sample_count": 1874, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1405840, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:44:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.669, "last_frame_pushed_at": "2026-07-01T08:44:36+09:00", "last_frame_read_age_sec": 4.833, "last_frame_read_at": "2026-07-01T08:44:36+09:00", "mjpeg": {"age_sec": 4.651, "exists": true, "mtime": "2026-07-01T08:44:36.665435+09:00", "mtime_epoch": 1782863076.665435, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164116}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:44:41+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5035.898, "name": "celery", "pid": 267, "rss_mb": 5035.898, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 44, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.467, "pushed": 1.467, "read": 1.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.586, "count": 44, "max_ms": 34.488, "p95_ms": 32.774}, "gstreamer_push": {"avg_ms": 1.062, "count": 44, "max_ms": 1.183, "p95_ms": 1.149}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.412, "count": 44, "max_ms": 41.453, "p95_ms": 40.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.921, "count": 44, "max_ms": 85.624, "p95_ms": 82.406}, "yolo11_640_fire_11class": {"avg_ms": 50.034, "count": 44, "max_ms": 58.292, "p95_ms": 56.821}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.267, "count": 44, "max_ms": 78.44, "p95_ms": 75.925}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.846, "count": 44, "max_ms": 162.103, "p95_ms": 152.983}, "read_interval": {"avg_ms": 575.578, "count": 44, "max_ms": 18765.028, "p95_ms": 164.533}, "triton_wait": {"avg_ms": 70.933, "count": 44, "max_ms": 85.637, "p95_ms": 82.418}, "visualize": {"avg_ms": 4.287, "count": 44, "max_ms": 5.076, "p95_ms": 4.483}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11724}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10820.535, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562cd3513a40] error while decoding MB 1 7, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 452, "h264_decode_error_count": 1364, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:47:22+09:00", "last_error_summary": "[h264 @ 0x562cd3513a40] error while decoding MB 1 7, bytestream -7", "last_exit_at": "2026-07-01T08:44:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.069, "last_started_at": "2026-07-01T08:44:39+09:00", "pid": 90945, "process": {"exists": false, "pid": 90945}, "restart_count": 496, "restart_count_per_hour": 21, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 497}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.92, "sample_count": 1903, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1406956, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:47:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T08:47:22+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T08:47:22+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T08:47:22.341626+09:00", "mtime_epoch": 1782863242.341626, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168890}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:47:22+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5036.414, "name": "celery", "pid": 267, "rss_mb": 5036.414, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 212, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.069, "pushed": 7.069, "read": 7.069}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.302, "count": 211, "max_ms": 39.062, "p95_ms": 34.081}, "gstreamer_push": {"avg_ms": 1.06, "count": 212, "max_ms": 1.298, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.29, "count": 211, "max_ms": 41.035, "p95_ms": 33.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.089, "count": 211, "max_ms": 86.881, "p95_ms": 80.849}, "yolo11_640_fire_11class": {"avg_ms": 46.406, "count": 211, "max_ms": 63.95, "p95_ms": 58.146}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.122, "count": 211, "max_ms": 84.816, "p95_ms": 75.615}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.937, "count": 212, "max_ms": 160.002, "p95_ms": 150.2}, "read_interval": {"avg_ms": 141.903, "count": 212, "max_ms": 173.376, "p95_ms": 162.202}, "triton_wait": {"avg_ms": 60.146, "count": 211, "max_ms": 86.894, "p95_ms": 80.862}, "visualize": {"avg_ms": 4.242, "count": 211, "max_ms": 5.635, "p95_ms": 4.639}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11724}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10819.855, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:47:22+09:00", "last_error_summary": "[h264 @ 0x562cd3513a40] error while decoding MB 1 7, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 452, "h264_decode_error_count": 1364, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:47:22+09:00", "last_error_summary": "[h264 @ 0x562cd3513a40] error while decoding MB 1 7, bytestream -7", "last_exit_at": "2026-07-01T08:47:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.159, "last_started_at": "2026-07-01T08:44:39+09:00", "pid": 90945, "process": {"exists": false, "pid": 90945}, "restart_count": 496, "restart_count_per_hour": 21, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 497}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.92, "sample_count": 1903, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1406957, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:47:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T08:47:22+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-01T08:47:22+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-01T08:47:22.341626+09:00", "mtime_epoch": 1782863242.341626, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168890}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:47:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5036.414, "name": "celery", "pid": 267, "rss_mb": 5036.414, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.082, "pushed": 7.082, "read": 7.048}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.318, "count": 212, "max_ms": 39.062, "p95_ms": 34.092}, "gstreamer_push": {"avg_ms": 1.06, "count": 212, "max_ms": 1.298, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.285, "count": 212, "max_ms": 41.035, "p95_ms": 33.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.071, "count": 212, "max_ms": 86.881, "p95_ms": 80.849}, "yolo11_640_fire_11class": {"avg_ms": 46.394, "count": 212, "max_ms": 63.95, "p95_ms": 58.146}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.117, "count": 212, "max_ms": 84.816, "p95_ms": 75.615}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.878, "count": 212, "max_ms": 160.002, "p95_ms": 150.2}, "read_interval": {"avg_ms": 141.837, "count": 211, "max_ms": 173.376, "p95_ms": 162.202}, "triton_wait": {"avg_ms": 60.129, "count": 212, "max_ms": 86.894, "p95_ms": 80.862}, "visualize": {"avg_ms": 4.242, "count": 212, "max_ms": 5.635, "p95_ms": 4.639}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11724}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10819.848, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 452, "h264_decode_error_count": 1364, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:47:22+09:00", "last_error_summary": "[h264 @ 0x562cd3513a40] error while decoding MB 1 7, bytestream -7", "last_exit_at": "2026-07-01T08:47:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.164, "last_started_at": "2026-07-01T08:44:39+09:00", "pid": 90945, "process": {"exists": false, "pid": 90945}, "restart_count": 496, "restart_count_per_hour": 21, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 497}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.92, "sample_count": 1903, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1406957, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:47:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T08:47:22+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-01T08:47:22+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-01T08:47:22.341626+09:00", "mtime_epoch": 1782863242.341626, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168890}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:47:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5036.414, "name": "celery", "pid": 267, "rss_mb": 5036.414, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.081, "pushed": 7.081, "read": 7.047}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.318, "count": 212, "max_ms": 39.062, "p95_ms": 34.092}, "gstreamer_push": {"avg_ms": 1.06, "count": 212, "max_ms": 1.298, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.285, "count": 212, "max_ms": 41.035, "p95_ms": 33.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.071, "count": 212, "max_ms": 86.881, "p95_ms": 80.849}, "yolo11_640_fire_11class": {"avg_ms": 46.394, "count": 212, "max_ms": 63.95, "p95_ms": 58.146}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.117, "count": 212, "max_ms": 84.816, "p95_ms": 75.615}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.878, "count": 212, "max_ms": 160.002, "p95_ms": 150.2}, "read_interval": {"avg_ms": 141.837, "count": 211, "max_ms": 173.376, "p95_ms": 162.202}, "triton_wait": {"avg_ms": 60.129, "count": 212, "max_ms": 86.894, "p95_ms": 80.862}, "visualize": {"avg_ms": 4.242, "count": 212, "max_ms": 5.635, "p95_ms": 4.639}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11724}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10927.766, "limit_mb": 8796093022207.996}, "detail": {"pid": 91052, "restart_count": 497, "start_count": 498}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 452, "h264_decode_error_count": 1364, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:47:22+09:00", "last_error_summary": "[h264 @ 0x562cd3513a40] error while decoding MB 1 7, bytestream -7", "last_exit_at": "2026-07-01T08:47:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:47:24+09:00", "pid": 91052, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.812, "name": "ffmpeg", "pid": 91052, "rss_mb": 138.812, "state": "S (sleeping)", "threads": 106}, "restart_count": 497, "restart_count_per_hour": 22, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 498}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.92, "sample_count": 1873, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1406957, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:47:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.204, "last_frame_pushed_at": "2026-07-01T08:47:22+09:00", "last_frame_read_age_sec": 4.337, "last_frame_read_at": "2026-07-01T08:47:22+09:00", "mjpeg": {"age_sec": 4.186, "exists": true, "mtime": "2026-07-01T08:47:22.485630+09:00", "mtime_epoch": 1782863242.4856296, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168666}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:47:26+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5036.414, "name": "celery", "pid": 267, "rss_mb": 5036.414, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 182, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.12, "pushed": 6.12, "read": 6.087}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.317, "count": 183, "max_ms": 39.062, "p95_ms": 34.081}, "gstreamer_push": {"avg_ms": 1.055, "count": 183, "max_ms": 1.298, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.11, "count": 183, "max_ms": 41.035, "p95_ms": 33.7}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.509, "count": 183, "max_ms": 86.881, "p95_ms": 80.849}, "yolo11_640_fire_11class": {"avg_ms": 45.806, "count": 183, "max_ms": 63.95, "p95_ms": 57.003}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.659, "count": 183, "max_ms": 84.816, "p95_ms": 77.576}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.324, "count": 183, "max_ms": 160.002, "p95_ms": 150.287}, "read_interval": {"avg_ms": 141.244, "count": 182, "max_ms": 173.376, "p95_ms": 162.202}, "triton_wait": {"avg_ms": 59.573, "count": 183, "max_ms": 86.894, "p95_ms": 80.862}, "visualize": {"avg_ms": 4.242, "count": 183, "max_ms": 5.635, "p95_ms": 4.654}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11733}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10821.441, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559c9420fe40] error while decoding MB 8 25, bytestream -20", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 453, "h264_decode_error_count": 1365, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:50:07+09:00", "last_error_summary": "[h264 @ 0x559c9420fe40] error while decoding MB 8 25, bytestream -20", "last_exit_at": "2026-07-01T08:47:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.274, "last_started_at": "2026-07-01T08:47:24+09:00", "pid": 91052, "process": {"exists": false, "pid": 91052}, "restart_count": 497, "restart_count_per_hour": 21, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 498}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.626, "sample_count": 2041, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1408070, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:50:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T08:50:07+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T08:50:07+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T08:50:07.901516+09:00", "mtime_epoch": 1782863407.9015162, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169104}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:50:07+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5036.93, "name": "celery", "pid": 267, "rss_mb": 5036.93, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.904, "pushed": 6.904, "read": 6.904}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.654, "count": 206, "max_ms": 41.375, "p95_ms": 34.894}, "gstreamer_push": {"avg_ms": 1.087, "count": 207, "max_ms": 1.363, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.074, "count": 206, "max_ms": 40.846, "p95_ms": 34.137}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.99, "count": 206, "max_ms": 87.311, "p95_ms": 83.449}, "yolo11_640_fire_11class": {"avg_ms": 48.383, "count": 206, "max_ms": 67.592, "p95_ms": 63.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.752, "count": 206, "max_ms": 85.947, "p95_ms": 80.688}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.354, "count": 207, "max_ms": 159.662, "p95_ms": 152.84}, "read_interval": {"avg_ms": 145.288, "count": 207, "max_ms": 175.833, "p95_ms": 165.878}, "triton_wait": {"avg_ms": 63.169, "count": 206, "max_ms": 87.323, "p95_ms": 83.461}, "visualize": {"avg_ms": 4.282, "count": 206, "max_ms": 5.306, "p95_ms": 4.527}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11733}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10820.816, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:50:07+09:00", "last_error_summary": "[h264 @ 0x559c9420fe40] error while decoding MB 8 25, bytestream -20", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 453, "h264_decode_error_count": 1365, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:50:07+09:00", "last_error_summary": "[h264 @ 0x559c9420fe40] error while decoding MB 8 25, bytestream -20", "last_exit_at": "2026-07-01T08:50:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.361, "last_started_at": "2026-07-01T08:47:24+09:00", "pid": 91052, "process": {"exists": false, "pid": 91052}, "restart_count": 497, "restart_count_per_hour": 21, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 498}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.626, "sample_count": 2042, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1408071, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:50:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:50:08+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-01T08:50:07+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T08:50:07.901516+09:00", "mtime_epoch": 1782863407.9015162, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169104}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:50:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5036.93, "name": "celery", "pid": 267, "rss_mb": 5036.93, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.916, "pushed": 6.916, "read": 6.883}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.65, "count": 207, "max_ms": 41.375, "p95_ms": 34.894}, "gstreamer_push": {"avg_ms": 1.087, "count": 207, "max_ms": 1.363, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.074, "count": 207, "max_ms": 40.846, "p95_ms": 34.137}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.985, "count": 207, "max_ms": 87.311, "p95_ms": 83.449}, "yolo11_640_fire_11class": {"avg_ms": 48.39, "count": 207, "max_ms": 67.592, "p95_ms": 63.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.746, "count": 207, "max_ms": 85.947, "p95_ms": 80.688}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.308, "count": 207, "max_ms": 159.662, "p95_ms": 152.84}, "read_interval": {"avg_ms": 145.235, "count": 206, "max_ms": 175.833, "p95_ms": 165.878}, "triton_wait": {"avg_ms": 63.163, "count": 207, "max_ms": 87.323, "p95_ms": 83.461}, "visualize": {"avg_ms": 4.282, "count": 207, "max_ms": 5.306, "p95_ms": 4.527}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11733}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10820.809, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 453, "h264_decode_error_count": 1365, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:50:07+09:00", "last_error_summary": "[h264 @ 0x559c9420fe40] error while decoding MB 8 25, bytestream -20", "last_exit_at": "2026-07-01T08:50:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.366, "last_started_at": "2026-07-01T08:47:24+09:00", "pid": 91052, "process": {"exists": false, "pid": 91052}, "restart_count": 497, "restart_count_per_hour": 21, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 498}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.626, "sample_count": 2042, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1408071, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:50:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:50:08+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-01T08:50:07+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T08:50:07.901516+09:00", "mtime_epoch": 1782863407.9015162, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169104}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:50:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5036.93, "name": "celery", "pid": 267, "rss_mb": 5036.93, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 207, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.915, "pushed": 6.915, "read": 6.882}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.65, "count": 207, "max_ms": 41.375, "p95_ms": 34.894}, "gstreamer_push": {"avg_ms": 1.087, "count": 207, "max_ms": 1.363, "p95_ms": 1.161}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.074, "count": 207, "max_ms": 40.846, "p95_ms": 34.137}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.985, "count": 207, "max_ms": 87.311, "p95_ms": 83.449}, "yolo11_640_fire_11class": {"avg_ms": 48.39, "count": 207, "max_ms": 67.592, "p95_ms": 63.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.746, "count": 207, "max_ms": 85.947, "p95_ms": 80.688}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.308, "count": 207, "max_ms": 159.662, "p95_ms": 152.84}, "read_interval": {"avg_ms": 145.235, "count": 206, "max_ms": 175.833, "p95_ms": 165.878}, "triton_wait": {"avg_ms": 63.163, "count": 207, "max_ms": 87.323, "p95_ms": 83.461}, "visualize": {"avg_ms": 4.282, "count": 207, "max_ms": 5.306, "p95_ms": 4.527}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11733}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10934.816, "limit_mb": 8796093022207.996}, "detail": {"pid": 91159, "restart_count": 498, "start_count": 499}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 453, "h264_decode_error_count": 1365, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:50:07+09:00", "last_error_summary": "[h264 @ 0x559c9420fe40] error while decoding MB 8 25, bytestream -20", "last_exit_at": "2026-07-01T08:50:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:50:10+09:00", "pid": 91159, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.93, "name": "ffmpeg", "pid": 91159, "rss_mb": 144.93, "state": "S (sleeping)", "threads": 106}, "restart_count": 498, "restart_count_per_hour": 22, "rtsp_timeout_count": 396, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 499}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 154.727, "sample_count": 2011, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1408071, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:50:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.244, "last_frame_pushed_at": "2026-07-01T08:50:08+09:00", "last_frame_read_age_sec": 4.377, "last_frame_read_at": "2026-07-01T08:50:07+09:00", "mjpeg": {"age_sec": 4.225, "exists": true, "mtime": "2026-07-01T08:50:08.045519+09:00", "mtime_epoch": 1782863408.0455194, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168990}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:50:12+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5036.93, "name": "celery", "pid": 267, "rss_mb": 5036.93, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 178, "pushed_frames": 178, "read_frames": 177, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.945, "pushed": 5.945, "read": 5.911}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.706, "count": 178, "max_ms": 41.375, "p95_ms": 35.981}, "gstreamer_push": {"avg_ms": 1.088, "count": 178, "max_ms": 1.363, "p95_ms": 1.167}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.971, "count": 178, "max_ms": 40.846, "p95_ms": 33.89}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.656, "count": 178, "max_ms": 87.311, "p95_ms": 83.46}, "yolo11_640_fire_11class": {"avg_ms": 48.354, "count": 178, "max_ms": 67.592, "p95_ms": 63.394}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.664, "count": 178, "max_ms": 85.947, "p95_ms": 80.956}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.101, "count": 178, "max_ms": 159.662, "p95_ms": 153.446}, "read_interval": {"avg_ms": 145.184, "count": 177, "max_ms": 175.833, "p95_ms": 166.006}, "triton_wait": {"avg_ms": 62.861, "count": 178, "max_ms": 87.323, "p95_ms": 83.472}, "visualize": {"avg_ms": 4.285, "count": 178, "max_ms": 5.306, "p95_ms": 4.536}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13252}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10765.562, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e52187acc0] error while decoding MB 9 43, bytestream -5 [in#0/rtsp @ 0x55e520f7ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e520f7ae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 203, "h264_decode_error_count": 209, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:52:19+09:00", "last_error_summary": "[h264 @ 0x55e52187acc0] error while decoding MB 9 43, bytestream -5 [in#0/rtsp @ 0x55e520f7ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e520f7ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:41:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 646.579, "last_started_at": "2026-07-01T08:41:33+09:00", "pid": 90686, "process": {"exists": false, "pid": 90686}, "restart_count": 202, "restart_count_per_hour": 4, "rtsp_timeout_count": 332, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 203}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.787, "sample_count": 2579, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1590245, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:52:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.085, "last_frame_pushed_at": "2026-07-01T08:52:19+09:00", "last_frame_read_age_sec": 0.071, "last_frame_read_at": "2026-07-01T08:52:19+09:00", "mjpeg": {"age_sec": 0.069, "exists": true, "mtime": "2026-07-01T08:52:19.792458+09:00", "mtime_epoch": 1782863539.7924578, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217643}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:52:19+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1810.457, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 107, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.568, "pushed": 3.568, "read": 3.568}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.037, "count": 107, "max_ms": 26.262, "p95_ms": 22.483}, "gstreamer_push": {"avg_ms": 1.166, "count": 107, "max_ms": 1.51, "p95_ms": 1.266}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.578, "count": 107, "max_ms": 54.705, "p95_ms": 48.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.131, "count": 107, "max_ms": 89.496, "p95_ms": 83.251}, "yolo11_640_fire_11class": {"avg_ms": 49.764, "count": 107, "max_ms": 70.627, "p95_ms": 61.504}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.15, "count": 107, "max_ms": 85.41, "p95_ms": 79.129}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.38, "count": 107, "max_ms": 116.902, "p95_ms": 111.312}, "read_interval": {"avg_ms": 280.373, "count": 107, "max_ms": 10160.556, "p95_ms": 126.076}, "triton_wait": {"avg_ms": 59.492, "count": 107, "max_ms": 89.509, "p95_ms": 83.263}, "visualize": {"avg_ms": 3.344, "count": 107, "max_ms": 4.129, "p95_ms": 3.613}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13252}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10765.203, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:52:19+09:00", "last_error_summary": "[h264 @ 0x55e52187acc0] error while decoding MB 9 43, bytestream -5 [in#0/rtsp @ 0x55e520f7ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e520f7ae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 203, "h264_decode_error_count": 209, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:52:19+09:00", "last_error_summary": "[h264 @ 0x55e52187acc0] error while decoding MB 9 43, bytestream -5 [in#0/rtsp @ 0x55e520f7ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e520f7ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:52:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 646.602, "last_started_at": "2026-07-01T08:41:33+09:00", "pid": 90686, "process": {"exists": false, "pid": 90686}, "restart_count": 202, "restart_count_per_hour": 4, "rtsp_timeout_count": 332, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 203}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.787, "sample_count": 2580, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1590246, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:52:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T08:52:19+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T08:52:19+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-01T08:52:19.792458+09:00", "mtime_epoch": 1782863539.7924578, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217643}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:52:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1810.457, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 107, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.567, "pushed": 3.567, "read": 3.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.061, "count": 107, "max_ms": 26.262, "p95_ms": 22.483}, "gstreamer_push": {"avg_ms": 1.168, "count": 107, "max_ms": 1.51, "p95_ms": 1.269}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.556, "count": 107, "max_ms": 54.705, "p95_ms": 48.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.099, "count": 107, "max_ms": 89.496, "p95_ms": 83.251}, "yolo11_640_fire_11class": {"avg_ms": 49.761, "count": 107, "max_ms": 70.627, "p95_ms": 61.504}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.118, "count": 107, "max_ms": 85.41, "p95_ms": 79.129}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.387, "count": 107, "max_ms": 116.902, "p95_ms": 111.312}, "read_interval": {"avg_ms": 280.373, "count": 107, "max_ms": 10160.556, "p95_ms": 126.076}, "triton_wait": {"avg_ms": 59.46, "count": 107, "max_ms": 89.509, "p95_ms": 83.263}, "visualize": {"avg_ms": 3.353, "count": 107, "max_ms": 4.294, "p95_ms": 3.629}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13252}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10765.18, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 203, "h264_decode_error_count": 209, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:52:19+09:00", "last_error_summary": "[h264 @ 0x55e52187acc0] error while decoding MB 9 43, bytestream -5 [in#0/rtsp @ 0x55e520f7ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e520f7ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:52:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 646.61, "last_started_at": "2026-07-01T08:41:33+09:00", "pid": 90686, "process": {"exists": false, "pid": 90686}, "restart_count": 202, "restart_count_per_hour": 4, "rtsp_timeout_count": 332, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 203}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.787, "sample_count": 2580, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1590246, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:52:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T08:52:19+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-01T08:52:19+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-01T08:52:19.792458+09:00", "mtime_epoch": 1782863539.7924578, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217643}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:52:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1810.457, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.576, "pushed": 3.576, "read": 3.543}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.061, "count": 107, "max_ms": 26.262, "p95_ms": 22.483}, "gstreamer_push": {"avg_ms": 1.168, "count": 107, "max_ms": 1.51, "p95_ms": 1.269}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.556, "count": 107, "max_ms": 54.705, "p95_ms": 48.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.099, "count": 107, "max_ms": 89.496, "p95_ms": 83.251}, "yolo11_640_fire_11class": {"avg_ms": 49.761, "count": 107, "max_ms": 70.627, "p95_ms": 61.504}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.118, "count": 107, "max_ms": 85.41, "p95_ms": 79.129}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.387, "count": 107, "max_ms": 116.902, "p95_ms": 111.312}, "read_interval": {"avg_ms": 282.092, "count": 106, "max_ms": 10160.556, "p95_ms": 126.076}, "triton_wait": {"avg_ms": 59.46, "count": 107, "max_ms": 89.509, "p95_ms": 83.263}, "visualize": {"avg_ms": 3.353, "count": 107, "max_ms": 4.294, "p95_ms": 3.629}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13252}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10786.984, "limit_mb": 8796093022207.996}, "detail": {"pid": 91266, "restart_count": 203, "start_count": 204}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 203, "h264_decode_error_count": 209, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:52:19+09:00", "last_error_summary": "[h264 @ 0x55e52187acc0] error while decoding MB 9 43, bytestream -5 [in#0/rtsp @ 0x55e520f7ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e520f7ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:52:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:52:21+09:00", "pid": 91266, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.832, "name": "ffmpeg", "pid": 91266, "rss_mb": 57.832, "state": "S (sleeping)", "threads": 1}, "restart_count": 203, "restart_count_per_hour": 5, "rtsp_timeout_count": 332, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 204}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.784, "sample_count": 2549, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1590246, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:52:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.113, "last_frame_pushed_at": "2026-07-01T08:52:19+09:00", "last_frame_read_age_sec": 4.2, "last_frame_read_at": "2026-07-01T08:52:19+09:00", "mjpeg": {"age_sec": 4.091, "exists": true, "mtime": "2026-07-01T08:52:19.900460+09:00", "mtime_epoch": 1782863539.9004602, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217111}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:52:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1810.457, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 67, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.234, "pushed": 2.234, "read": 2.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.067, "count": 67, "max_ms": 26.262, "p95_ms": 22.542}, "gstreamer_push": {"avg_ms": 1.167, "count": 67, "max_ms": 1.51, "p95_ms": 1.269}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.21, "count": 67, "max_ms": 54.705, "p95_ms": 48.17}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.964, "count": 67, "max_ms": 84.89, "p95_ms": 83.251}, "yolo11_640_fire_11class": {"avg_ms": 48.863, "count": 67, "max_ms": 70.627, "p95_ms": 61.211}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.813, "count": 67, "max_ms": 81.063, "p95_ms": 79.129}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.02, "count": 67, "max_ms": 112.689, "p95_ms": 111.312}, "read_interval": {"avg_ms": 386.291, "count": 67, "max_ms": 10160.556, "p95_ms": 126.076}, "triton_wait": {"avg_ms": 59.099, "count": 67, "max_ms": 84.905, "p95_ms": 83.263}, "visualize": {"avg_ms": 3.352, "count": 67, "max_ms": 4.294, "p95_ms": 3.755}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11744}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10818.816, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 11, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561ae554e500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae554e500] error while decoding MB 0 38, bytestream 1333 [h264 @ 0x561ae61ec200] cabac decode of qscale diff failed at 57 4 [h264 @ 0x561ae61ec200] error while decoding MB 57 4, bytestream 6927 [h264 @ 0x561ae547f540] cabac decode of qscale diff failed at 63 25 [h264 @ 0x561ae547f540] error while decoding MB 63 25, bytestream 54048 [h264 @ 0x561ae543ed00] error while decoding MB 54 19, bytestream -32 [h264", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 454, "h264_decode_error_count": 1376, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:53:28+09:00", "last_error_summary": "[h264 @ 0x561ae554e500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae554e500] error while decoding MB 0 38, bytestream 1333 [h264 @ 0x561ae61ec200] cabac decode of qscale diff failed at 57 4 [h264 @ 0x561ae61ec200] error while decoding MB 57 4, bytestream 6927 [h264 @ 0x561ae547f540] cabac decode of qscale diff failed at 63 25 [h264 @ 0x561ae547f540] error while decoding MB 63 25, bytestream 54048 [h264 @ 0x561ae543ed00] error while decoding MB 54 19, bytestream -32 [h264", "last_exit_at": "2026-07-01T08:50:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 198.108, "last_started_at": "2026-07-01T08:50:10+09:00", "pid": 91159, "process": {"exists": false, "pid": 91159}, "restart_count": 498, "restart_count_per_hour": 21, "rtsp_timeout_count": 398, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 499}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.869, "sample_count": 1902, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1409296, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:53:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T08:53:28+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T08:53:28+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T08:53:28.333943+09:00", "mtime_epoch": 1782863608.3339434, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167787}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:53:28+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5037.703, "name": "celery", "pid": 267, "rss_mb": 5037.703, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 89, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.973, "pushed": 2.973, "read": 2.973}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.747, "count": 89, "max_ms": 34.205, "p95_ms": 32.585}, "gstreamer_push": {"avg_ms": 1.047, "count": 89, "max_ms": 1.702, "p95_ms": 1.123}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.045, "count": 89, "max_ms": 41.915, "p95_ms": 39.536}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.963, "count": 89, "max_ms": 84.132, "p95_ms": 82.718}, "yolo11_640_fire_11class": {"avg_ms": 51.484, "count": 89, "max_ms": 63.149, "p95_ms": 61.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.701, "count": 89, "max_ms": 81.567, "p95_ms": 80.187}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.514, "count": 89, "max_ms": 154.996, "p95_ms": 152.06}, "read_interval": {"avg_ms": 337.299, "count": 89, "max_ms": 9082.343, "p95_ms": 165.331}, "triton_wait": {"avg_ms": 69.18, "count": 89, "max_ms": 84.144, "p95_ms": 82.73}, "visualize": {"avg_ms": 4.283, "count": 89, "max_ms": 5.423, "p95_ms": 4.638}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11744}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10818.438, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:53:28+09:00", "last_error_summary": "[h264 @ 0x561ae554e500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae554e500] error while decoding MB 0 38, bytestream 1333 [h264 @ 0x561ae61ec200] cabac decode of qscale diff failed at 57 4 [h264 @ 0x561ae61ec200] error while decoding MB 57 4, bytestream 6927 [h264 @ 0x561ae547f540] cabac decode of qscale diff failed at 63 25 [h264 @ 0x561ae547f540] error while decoding MB 63 25, bytestream 54048 [h264 @ 0x561ae543ed00] error while decoding MB 54 19, bytestream -32 [h264", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 454, "h264_decode_error_count": 1376, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:53:28+09:00", "last_error_summary": "[h264 @ 0x561ae554e500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae554e500] error while decoding MB 0 38, bytestream 1333 [h264 @ 0x561ae61ec200] cabac decode of qscale diff failed at 57 4 [h264 @ 0x561ae61ec200] error while decoding MB 57 4, bytestream 6927 [h264 @ 0x561ae547f540] cabac decode of qscale diff failed at 63 25 [h264 @ 0x561ae547f540] error while decoding MB 63 25, bytestream 54048 [h264 @ 0x561ae543ed00] error while decoding MB 54 19, bytestream -32 [h264", "last_exit_at": "2026-07-01T08:53:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 198.198, "last_started_at": "2026-07-01T08:50:10+09:00", "pid": 91159, "process": {"exists": false, "pid": 91159}, "restart_count": 498, "restart_count_per_hour": 21, "rtsp_timeout_count": 398, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 499}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.869, "sample_count": 1902, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1409297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:53:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T08:53:28+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-01T08:53:28+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-01T08:53:28.333943+09:00", "mtime_epoch": 1782863608.3339434, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167787}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:53:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5037.703, "name": "celery", "pid": 267, "rss_mb": 5037.703, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 88, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.977, "pushed": 2.977, "read": 2.943}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.789, "count": 89, "max_ms": 36.542, "p95_ms": 32.585}, "gstreamer_push": {"avg_ms": 1.047, "count": 89, "max_ms": 1.702, "p95_ms": 1.123}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.017, "count": 89, "max_ms": 41.915, "p95_ms": 39.536}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.922, "count": 89, "max_ms": 84.132, "p95_ms": 82.718}, "yolo11_640_fire_11class": {"avg_ms": 51.47, "count": 89, "max_ms": 63.149, "p95_ms": 61.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.693, "count": 89, "max_ms": 81.567, "p95_ms": 80.187}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.578, "count": 89, "max_ms": 154.996, "p95_ms": 152.06}, "read_interval": {"avg_ms": 339.508, "count": 88, "max_ms": 9082.343, "p95_ms": 165.331}, "triton_wait": {"avg_ms": 69.14, "count": 89, "max_ms": 84.144, "p95_ms": 82.73}, "visualize": {"avg_ms": 4.293, "count": 89, "max_ms": 5.423, "p95_ms": 4.653}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11744}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10818.43, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 454, "h264_decode_error_count": 1376, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:53:28+09:00", "last_error_summary": "[h264 @ 0x561ae554e500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae554e500] error while decoding MB 0 38, bytestream 1333 [h264 @ 0x561ae61ec200] cabac decode of qscale diff failed at 57 4 [h264 @ 0x561ae61ec200] error while decoding MB 57 4, bytestream 6927 [h264 @ 0x561ae547f540] cabac decode of qscale diff failed at 63 25 [h264 @ 0x561ae547f540] error while decoding MB 63 25, bytestream 54048 [h264 @ 0x561ae543ed00] error while decoding MB 54 19, bytestream -32 [h264", "last_exit_at": "2026-07-01T08:53:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 198.203, "last_started_at": "2026-07-01T08:50:10+09:00", "pid": 91159, "process": {"exists": false, "pid": 91159}, "restart_count": 498, "restart_count_per_hour": 21, "rtsp_timeout_count": 398, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 499}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.869, "sample_count": 1902, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1409297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:53:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T08:53:28+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-01T08:53:28+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-01T08:53:28.333943+09:00", "mtime_epoch": 1782863608.3339434, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167787}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:53:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5037.703, "name": "celery", "pid": 267, "rss_mb": 5037.703, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 88, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.976, "pushed": 2.976, "read": 2.943}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.789, "count": 89, "max_ms": 36.542, "p95_ms": 32.585}, "gstreamer_push": {"avg_ms": 1.047, "count": 89, "max_ms": 1.702, "p95_ms": 1.123}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.017, "count": 89, "max_ms": 41.915, "p95_ms": 39.536}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.922, "count": 89, "max_ms": 84.132, "p95_ms": 82.718}, "yolo11_640_fire_11class": {"avg_ms": 51.47, "count": 89, "max_ms": 63.149, "p95_ms": 61.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.693, "count": 89, "max_ms": 81.567, "p95_ms": 80.187}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.578, "count": 89, "max_ms": 154.996, "p95_ms": 152.06}, "read_interval": {"avg_ms": 339.508, "count": 88, "max_ms": 9082.343, "p95_ms": 165.331}, "triton_wait": {"avg_ms": 69.14, "count": 89, "max_ms": 84.144, "p95_ms": 82.73}, "visualize": {"avg_ms": 4.293, "count": 89, "max_ms": 5.423, "p95_ms": 4.653}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11744}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10934.852, "limit_mb": 8796093022207.996}, "detail": {"pid": 91418, "restart_count": 499, "start_count": 500}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 454, "h264_decode_error_count": 1376, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:53:28+09:00", "last_error_summary": "[h264 @ 0x561ae554e500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae554e500] error while decoding MB 0 38, bytestream 1333 [h264 @ 0x561ae61ec200] cabac decode of qscale diff failed at 57 4 [h264 @ 0x561ae61ec200] error while decoding MB 57 4, bytestream 6927 [h264 @ 0x561ae547f540] cabac decode of qscale diff failed at 63 25 [h264 @ 0x561ae547f540] error while decoding MB 63 25, bytestream 54048 [h264 @ 0x561ae543ed00] error while decoding MB 54 19, bytestream -32 [h264", "last_exit_at": "2026-07-01T08:53:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:53:31+09:00", "pid": 91418, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.387, "name": "ffmpeg", "pid": 91418, "rss_mb": 147.387, "state": "S (sleeping)", "threads": 106}, "restart_count": 499, "restart_count_per_hour": 22, "rtsp_timeout_count": 398, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 500}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.948, "sample_count": 1869, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1409297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:53:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.914, "last_frame_pushed_at": "2026-07-01T08:53:28+09:00", "last_frame_read_age_sec": 5.053, "last_frame_read_at": "2026-07-01T08:53:28+09:00", "mjpeg": {"age_sec": 4.9, "exists": true, "mtime": "2026-07-01T08:53:28.477947+09:00", "mtime_epoch": 1782863608.4779465, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165501}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:53:33+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5037.703, "name": "celery", "pid": 267, "rss_mb": 5037.703, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.967, "pushed": 1.967, "read": 1.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.999, "count": 58, "max_ms": 36.542, "p95_ms": 33.336}, "gstreamer_push": {"avg_ms": 1.023, "count": 59, "max_ms": 1.509, "p95_ms": 1.123}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.586, "count": 58, "max_ms": 41.915, "p95_ms": 39.366}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.374, "count": 58, "max_ms": 81.663, "p95_ms": 79.899}, "yolo11_640_fire_11class": {"avg_ms": 48.76, "count": 58, "max_ms": 61.112, "p95_ms": 60.051}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.968, "count": 58, "max_ms": 79.198, "p95_ms": 76.625}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.102, "count": 59, "max_ms": 152.06, "p95_ms": 151.18}, "read_interval": {"avg_ms": 432.61, "count": 58, "max_ms": 9082.343, "p95_ms": 164.131}, "triton_wait": {"avg_ms": 64.396, "count": 58, "max_ms": 81.677, "p95_ms": 79.912}, "visualize": {"avg_ms": 4.245, "count": 58, "max_ms": 5.296, "p95_ms": 4.782}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11753}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10821.891, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55cdd7512d80] error while decoding MB 25 17, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 455, "h264_decode_error_count": 1377, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:56:15+09:00", "last_error_summary": "[h264 @ 0x55cdd7512d80] error while decoding MB 25 17, bytestream -7", "last_exit_at": "2026-07-01T08:53:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.12, "last_started_at": "2026-07-01T08:53:31+09:00", "pid": 91418, "process": {"exists": false, "pid": 91418}, "restart_count": 499, "restart_count_per_hour": 21, "rtsp_timeout_count": 398, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 500}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.786, "sample_count": 1900, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1410391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:56:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T08:56:15+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T08:56:15+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T08:56:15.449468+09:00", "mtime_epoch": 1782863775.4494677, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165371}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:56:15+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5038.219, "name": "celery", "pid": 267, "rss_mb": 5038.219, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.708, "pushed": 6.708, "read": 6.708}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.168, "count": 200, "max_ms": 39.256, "p95_ms": 34.653}, "gstreamer_push": {"avg_ms": 1.042, "count": 201, "max_ms": 1.247, "p95_ms": 1.136}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.774, "count": 200, "max_ms": 40.98, "p95_ms": 36.304}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.409, "count": 200, "max_ms": 92.619, "p95_ms": 83.834}, "yolo11_640_fire_11class": {"avg_ms": 50.489, "count": 200, "max_ms": 68.657, "p95_ms": 62.647}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.751, "count": 200, "max_ms": 88.071, "p95_ms": 79.05}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.136, "count": 201, "max_ms": 183.866, "p95_ms": 155.966}, "read_interval": {"avg_ms": 149.474, "count": 201, "max_ms": 195.175, "p95_ms": 169.081}, "triton_wait": {"avg_ms": 67.581, "count": 200, "max_ms": 92.631, "p95_ms": 84.388}, "visualize": {"avg_ms": 4.358, "count": 200, "max_ms": 5.907, "p95_ms": 5.052}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11753}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10821.258, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:56:15+09:00", "last_error_summary": "[h264 @ 0x55cdd7512d80] error while decoding MB 25 17, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 455, "h264_decode_error_count": 1377, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:56:15+09:00", "last_error_summary": "[h264 @ 0x55cdd7512d80] error while decoding MB 25 17, bytestream -7", "last_exit_at": "2026-07-01T08:56:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.222, "last_started_at": "2026-07-01T08:53:31+09:00", "pid": 91418, "process": {"exists": false, "pid": 91418}, "restart_count": 499, "restart_count_per_hour": 21, "rtsp_timeout_count": 398, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 500}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.786, "sample_count": 1900, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1410392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:56:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T08:56:15+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-01T08:56:15+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-01T08:56:15.449468+09:00", "mtime_epoch": 1782863775.4494677, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165371}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:56:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5038.219, "name": "celery", "pid": 267, "rss_mb": 5038.219, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.721, "pushed": 6.721, "read": 6.687}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.172, "count": 201, "max_ms": 39.256, "p95_ms": 34.653}, "gstreamer_push": {"avg_ms": 1.042, "count": 201, "max_ms": 1.247, "p95_ms": 1.136}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.765, "count": 201, "max_ms": 40.98, "p95_ms": 36.304}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.447, "count": 201, "max_ms": 92.619, "p95_ms": 83.834}, "yolo11_640_fire_11class": {"avg_ms": 50.53, "count": 201, "max_ms": 68.657, "p95_ms": 62.647}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.799, "count": 201, "max_ms": 88.071, "p95_ms": 79.05}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.223, "count": 201, "max_ms": 183.866, "p95_ms": 155.966}, "read_interval": {"avg_ms": 149.516, "count": 200, "max_ms": 195.175, "p95_ms": 169.081}, "triton_wait": {"avg_ms": 67.618, "count": 201, "max_ms": 92.631, "p95_ms": 84.388}, "visualize": {"avg_ms": 4.357, "count": 201, "max_ms": 5.907, "p95_ms": 5.052}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11753}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10821.25, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 455, "h264_decode_error_count": 1377, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:56:15+09:00", "last_error_summary": "[h264 @ 0x55cdd7512d80] error while decoding MB 25 17, bytestream -7", "last_exit_at": "2026-07-01T08:56:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.228, "last_started_at": "2026-07-01T08:53:31+09:00", "pid": 91418, "process": {"exists": false, "pid": 91418}, "restart_count": 499, "restart_count_per_hour": 21, "rtsp_timeout_count": 398, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 500}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.786, "sample_count": 1900, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1410392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:56:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T08:56:15+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-01T08:56:15+09:00", "mjpeg": {"age_sec": 0.155, "exists": true, "mtime": "2026-07-01T08:56:15.449468+09:00", "mtime_epoch": 1782863775.4494677, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165371}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:56:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5038.219, "name": "celery", "pid": 267, "rss_mb": 5038.219, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.72, "pushed": 6.72, "read": 6.686}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.172, "count": 201, "max_ms": 39.256, "p95_ms": 34.653}, "gstreamer_push": {"avg_ms": 1.042, "count": 201, "max_ms": 1.247, "p95_ms": 1.136}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.765, "count": 201, "max_ms": 40.98, "p95_ms": 36.304}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.447, "count": 201, "max_ms": 92.619, "p95_ms": 83.834}, "yolo11_640_fire_11class": {"avg_ms": 50.53, "count": 201, "max_ms": 68.657, "p95_ms": 62.647}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.799, "count": 201, "max_ms": 88.071, "p95_ms": 79.05}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.223, "count": 201, "max_ms": 183.866, "p95_ms": 155.966}, "read_interval": {"avg_ms": 149.516, "count": 200, "max_ms": 195.175, "p95_ms": 169.081}, "triton_wait": {"avg_ms": 67.618, "count": 201, "max_ms": 92.631, "p95_ms": 84.388}, "visualize": {"avg_ms": 4.357, "count": 201, "max_ms": 5.907, "p95_ms": 5.052}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11753}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10930.0, "limit_mb": 8796093022207.996}, "detail": {"pid": 91525, "restart_count": 500, "start_count": 501}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 455, "h264_decode_error_count": 1377, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:56:15+09:00", "last_error_summary": "[h264 @ 0x55cdd7512d80] error while decoding MB 25 17, bytestream -7", "last_exit_at": "2026-07-01T08:56:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:56:18+09:00", "pid": 91525, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.27, "name": "ffmpeg", "pid": 91525, "rss_mb": 139.27, "state": "S (sleeping)", "threads": 106}, "restart_count": 500, "restart_count_per_hour": 22, "rtsp_timeout_count": 398, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 501}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.606, "sample_count": 1870, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1410392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:56:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.414, "last_frame_pushed_at": "2026-07-01T08:56:15+09:00", "last_frame_read_age_sec": 4.563, "last_frame_read_at": "2026-07-01T08:56:15+09:00", "mjpeg": {"age_sec": 4.399, "exists": true, "mtime": "2026-07-01T08:56:15.609471+09:00", "mtime_epoch": 1782863775.609471, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163117}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:56:20+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5038.219, "name": "celery", "pid": 267, "rss_mb": 5038.219, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 170, "pushed_frames": 170, "read_frames": 169, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.677, "pushed": 5.677, "read": 5.643}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.121, "count": 170, "max_ms": 39.256, "p95_ms": 34.653}, "gstreamer_push": {"avg_ms": 1.052, "count": 170, "max_ms": 1.247, "p95_ms": 1.141}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.921, "count": 170, "max_ms": 40.98, "p95_ms": 36.879}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.089, "count": 170, "max_ms": 92.619, "p95_ms": 84.764}, "yolo11_640_fire_11class": {"avg_ms": 51.548, "count": 170, "max_ms": 68.657, "p95_ms": 63.679}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.366, "count": 170, "max_ms": 88.071, "p95_ms": 79.268}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.406, "count": 170, "max_ms": 183.866, "p95_ms": 157.607}, "read_interval": {"avg_ms": 150.951, "count": 169, "max_ms": 195.175, "p95_ms": 169.359}, "triton_wait": {"avg_ms": 69.289, "count": 170, "max_ms": 92.631, "p95_ms": 85.655}, "visualize": {"avg_ms": 4.36, "count": 170, "max_ms": 5.907, "p95_ms": 4.856}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13266}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10760.145, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557b425fd700] error while decoding MB 50 42, bytestream -13 [in#0/rtsp @ 0x557b41b64e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b41b64e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 204, "h264_decode_error_count": 210, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:56:59+09:00", "last_error_summary": "[h264 @ 0x557b425fd700] error while decoding MB 50 42, bytestream -13 [in#0/rtsp @ 0x557b41b64e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b41b64e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:52:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.219, "last_started_at": "2026-07-01T08:52:21+09:00", "pid": 91266, "process": {"exists": false, "pid": 91266}, "restart_count": 203, "restart_count_per_hour": 5, "rtsp_timeout_count": 334, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 204}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.8, "sample_count": 1794, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1592024, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:56:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.083, "last_frame_pushed_at": "2026-07-01T08:56:59+09:00", "last_frame_read_age_sec": 0.074, "last_frame_read_at": "2026-07-01T08:56:59+09:00", "mjpeg": {"age_sec": 0.067, "exists": true, "mtime": "2026-07-01T08:56:59.142370+09:00", "mtime_epoch": 1782863819.1423702, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 210244}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:56:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1811.23, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 66, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.203, "pushed": 2.203, "read": 2.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.518, "count": 66, "max_ms": 24.797, "p95_ms": 24.375}, "gstreamer_push": {"avg_ms": 1.052, "count": 66, "max_ms": 1.21, "p95_ms": 1.163}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.501, "count": 66, "max_ms": 56.13, "p95_ms": 52.737}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.322, "count": 66, "max_ms": 91.274, "p95_ms": 85.565}, "yolo11_640_fire_11class": {"avg_ms": 52.574, "count": 66, "max_ms": 66.78, "p95_ms": 63.32}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.802, "count": 66, "max_ms": 86.95, "p95_ms": 83.278}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.845, "count": 66, "max_ms": 157.313, "p95_ms": 151.029}, "read_interval": {"avg_ms": 455.025, "count": 66, "max_ms": 12364.296, "p95_ms": 163.99}, "triton_wait": {"avg_ms": 64.576, "count": 66, "max_ms": 91.286, "p95_ms": 85.578}, "visualize": {"avg_ms": 4.616, "count": 66, "max_ms": 5.409, "p95_ms": 5.137}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13266}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10759.527, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:56:59+09:00", "last_error_summary": "[h264 @ 0x557b425fd700] error while decoding MB 50 42, bytestream -13 [in#0/rtsp @ 0x557b41b64e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b41b64e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 204, "h264_decode_error_count": 210, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:56:59+09:00", "last_error_summary": "[h264 @ 0x557b425fd700] error while decoding MB 50 42, bytestream -13 [in#0/rtsp @ 0x557b41b64e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b41b64e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:56:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.281, "last_started_at": "2026-07-01T08:52:21+09:00", "pid": 91266, "process": {"exists": false, "pid": 91266}, "restart_count": 203, "restart_count_per_hour": 5, "rtsp_timeout_count": 334, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 204}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.8, "sample_count": 1794, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1592025, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:56:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T08:56:59+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-01T08:56:59+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-01T08:56:59.142370+09:00", "mtime_epoch": 1782863819.1423702, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 210244}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:56:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1811.23, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 65, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.21, "pushed": 2.21, "read": 2.176}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.545, "count": 66, "max_ms": 24.797, "p95_ms": 24.375}, "gstreamer_push": {"avg_ms": 1.053, "count": 66, "max_ms": 1.21, "p95_ms": 1.163}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.469, "count": 66, "max_ms": 56.13, "p95_ms": 52.737}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.912, "count": 66, "max_ms": 91.274, "p95_ms": 85.565}, "yolo11_640_fire_11class": {"avg_ms": 52.466, "count": 66, "max_ms": 66.78, "p95_ms": 63.32}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.445, "count": 66, "max_ms": 86.95, "p95_ms": 83.278}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.635, "count": 66, "max_ms": 157.313, "p95_ms": 151.029}, "read_interval": {"avg_ms": 459.669, "count": 65, "max_ms": 12364.296, "p95_ms": 163.99}, "triton_wait": {"avg_ms": 64.166, "count": 66, "max_ms": 91.286, "p95_ms": 85.578}, "visualize": {"avg_ms": 4.651, "count": 66, "max_ms": 6.803, "p95_ms": 5.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.865, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13266}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10759.473, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 204, "h264_decode_error_count": 210, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:56:59+09:00", "last_error_summary": "[h264 @ 0x557b425fd700] error while decoding MB 50 42, bytestream -13 [in#0/rtsp @ 0x557b41b64e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b41b64e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:56:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.286, "last_started_at": "2026-07-01T08:52:21+09:00", "pid": 91266, "process": {"exists": false, "pid": 91266}, "restart_count": 203, "restart_count_per_hour": 5, "rtsp_timeout_count": 334, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 204}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.8, "sample_count": 1794, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1592025, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:56:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T08:56:59+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-01T08:56:59+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-01T08:56:59.142370+09:00", "mtime_epoch": 1782863819.1423702, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 210244}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:56:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1811.23, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 65, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.21, "pushed": 2.21, "read": 2.176}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.545, "count": 66, "max_ms": 24.797, "p95_ms": 24.375}, "gstreamer_push": {"avg_ms": 1.053, "count": 66, "max_ms": 1.21, "p95_ms": 1.163}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.469, "count": 66, "max_ms": 56.13, "p95_ms": 52.737}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.912, "count": 66, "max_ms": 91.274, "p95_ms": 85.565}, "yolo11_640_fire_11class": {"avg_ms": 52.466, "count": 66, "max_ms": 66.78, "p95_ms": 63.32}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.445, "count": 66, "max_ms": 86.95, "p95_ms": 83.278}}, "truncated_count": 0}, "processing_total": {"avg_ms": 127.635, "count": 66, "max_ms": 157.313, "p95_ms": 151.029}, "read_interval": {"avg_ms": 459.669, "count": 65, "max_ms": 12364.296, "p95_ms": 163.99}, "triton_wait": {"avg_ms": 64.166, "count": 66, "max_ms": 91.286, "p95_ms": 85.578}, "visualize": {"avg_ms": 4.651, "count": 66, "max_ms": 6.803, "p95_ms": 5.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13266}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10875.918, "limit_mb": 8796093022207.996}, "detail": {"pid": 91632, "restart_count": 204, "start_count": 205}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 204, "h264_decode_error_count": 210, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:56:59+09:00", "last_error_summary": "[h264 @ 0x557b425fd700] error while decoding MB 50 42, bytestream -13 [in#0/rtsp @ 0x557b41b64e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557b41b64e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:57:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:57:02+09:00", "pid": 91632, "process": {"exists": true, "fd_count": 5, "hwm_mb": 148.695, "name": "ffmpeg", "pid": 91632, "rss_mb": 148.695, "state": "R (running)", "threads": 49}, "restart_count": 204, "restart_count_per_hour": 6, "rtsp_timeout_count": 334, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 205}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.8, "sample_count": 1792, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1592025, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:56:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.981, "last_frame_pushed_at": "2026-07-01T08:56:59+09:00", "last_frame_read_age_sec": 5.113, "last_frame_read_at": "2026-07-01T08:56:59+09:00", "mjpeg": {"age_sec": 4.962, "exists": true, "mtime": "2026-07-01T08:56:59.286373+09:00", "mtime_epoch": 1782863819.2863731, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 210026}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T08:57:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1811.23, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 31, "pushed_frames": 31, "read_frames": 30, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.035, "pushed": 1.035, "read": 1.001}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.011, "count": 31, "max_ms": 24.48, "p95_ms": 23.928}, "gstreamer_push": {"avg_ms": 1.057, "count": 31, "max_ms": 1.21, "p95_ms": 1.175}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.637, "count": 31, "max_ms": 54.196, "p95_ms": 51.126}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.187, "count": 31, "max_ms": 82.294, "p95_ms": 82.219}, "yolo11_640_fire_11class": {"avg_ms": 50.78, "count": 31, "max_ms": 63.32, "p95_ms": 63.128}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.178, "count": 31, "max_ms": 78.529, "p95_ms": 76.697}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.866, "count": 31, "max_ms": 146.827, "p95_ms": 146.108}, "read_interval": {"avg_ms": 832.348, "count": 30, "max_ms": 12364.296, "p95_ms": 8809.727}, "triton_wait": {"avg_ms": 61.692, "count": 31, "max_ms": 82.306, "p95_ms": 82.231}, "visualize": {"avg_ms": 4.678, "count": 31, "max_ms": 6.803, "p95_ms": 5.409}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11762}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10824.883, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 0, "rtsp_timeout_count": 2, "sample_message": "[in#0/rtsp @ 0x560ac173fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ac173fe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 456, "h264_decode_error_count": 1377, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:59:11+09:00", "last_error_summary": "[in#0/rtsp @ 0x560ac173fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ac173fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:56:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 173.814, "last_started_at": "2026-07-01T08:56:18+09:00", "pid": 91525, "process": {"exists": false, "pid": 91525}, "restart_count": 500, "restart_count_per_hour": 21, "rtsp_timeout_count": 400, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 501}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.859, "sample_count": 1897, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1411462, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:59:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T08:59:11+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T08:59:11+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T08:59:11.773071+09:00", "mtime_epoch": 1782863951.7730708, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162617}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:59:11+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5038.734, "name": "celery", "pid": 267, "rss_mb": 5038.734, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 77, "pushed_frames": 77, "read_frames": 77, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.57, "pushed": 2.57, "read": 2.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.11, "count": 77, "max_ms": 35.206, "p95_ms": 33.934}, "gstreamer_push": {"avg_ms": 1.06, "count": 77, "max_ms": 1.31, "p95_ms": 1.275}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.624, "count": 77, "max_ms": 42.27, "p95_ms": 39.629}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.575, "count": 77, "max_ms": 88.409, "p95_ms": 87.074}, "yolo11_640_fire_11class": {"avg_ms": 47.444, "count": 77, "max_ms": 66.982, "p95_ms": 64.414}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.172, "count": 77, "max_ms": 86.6, "p95_ms": 82.369}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.649, "count": 77, "max_ms": 157.971, "p95_ms": 155.061}, "read_interval": {"avg_ms": 390.09, "count": 77, "max_ms": 10136.602, "p95_ms": 168.82}, "triton_wait": {"avg_ms": 63.646, "count": 77, "max_ms": 88.421, "p95_ms": 87.086}, "visualize": {"avg_ms": 4.247, "count": 77, "max_ms": 5.485, "p95_ms": 4.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11762}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10824.367, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T08:59:11+09:00", "last_error_summary": "[in#0/rtsp @ 0x560ac173fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ac173fe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 456, "h264_decode_error_count": 1377, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:59:11+09:00", "last_error_summary": "[in#0/rtsp @ 0x560ac173fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ac173fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:59:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 173.916, "last_started_at": "2026-07-01T08:56:18+09:00", "pid": 91525, "process": {"exists": false, "pid": 91525}, "restart_count": 500, "restart_count_per_hour": 21, "rtsp_timeout_count": 400, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 501}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.859, "sample_count": 1897, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1411463, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:59:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T08:59:11+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-01T08:59:11+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-01T08:59:11.773071+09:00", "mtime_epoch": 1782863951.7730708, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162617}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:59:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5038.734, "name": "celery", "pid": 267, "rss_mb": 5038.734, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 77, "pushed_frames": 77, "read_frames": 76, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.573, "pushed": 2.573, "read": 2.54}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.09, "count": 77, "max_ms": 35.206, "p95_ms": 33.934}, "gstreamer_push": {"avg_ms": 1.061, "count": 77, "max_ms": 1.31, "p95_ms": 1.275}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.757, "count": 77, "max_ms": 42.27, "p95_ms": 40.513}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.902, "count": 77, "max_ms": 88.409, "p95_ms": 87.074}, "yolo11_640_fire_11class": {"avg_ms": 47.664, "count": 77, "max_ms": 66.982, "p95_ms": 64.414}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.574, "count": 77, "max_ms": 86.6, "p95_ms": 82.369}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.965, "count": 77, "max_ms": 157.971, "p95_ms": 155.061}, "read_interval": {"avg_ms": 393.397, "count": 76, "max_ms": 10136.602, "p95_ms": 168.82}, "triton_wait": {"avg_ms": 63.973, "count": 77, "max_ms": 88.421, "p95_ms": 87.086}, "visualize": {"avg_ms": 4.25, "count": 77, "max_ms": 5.485, "p95_ms": 4.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11762}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10824.332, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 456, "h264_decode_error_count": 1377, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:59:11+09:00", "last_error_summary": "[in#0/rtsp @ 0x560ac173fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ac173fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:59:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 173.92, "last_started_at": "2026-07-01T08:56:18+09:00", "pid": 91525, "process": {"exists": false, "pid": 91525}, "restart_count": 500, "restart_count_per_hour": 21, "rtsp_timeout_count": 400, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 501}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.859, "sample_count": 1897, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1411463, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:59:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T08:59:11+09:00", "last_frame_read_age_sec": 0.16, "last_frame_read_at": "2026-07-01T08:59:11+09:00", "mjpeg": {"age_sec": 0.155, "exists": true, "mtime": "2026-07-01T08:59:11.773071+09:00", "mtime_epoch": 1782863951.7730708, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162617}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:59:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5038.734, "name": "celery", "pid": 267, "rss_mb": 5038.734, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 77, "pushed_frames": 77, "read_frames": 76, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.573, "pushed": 2.573, "read": 2.539}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.09, "count": 77, "max_ms": 35.206, "p95_ms": 33.934}, "gstreamer_push": {"avg_ms": 1.061, "count": 77, "max_ms": 1.31, "p95_ms": 1.275}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.757, "count": 77, "max_ms": 42.27, "p95_ms": 40.513}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.902, "count": 77, "max_ms": 88.409, "p95_ms": 87.074}, "yolo11_640_fire_11class": {"avg_ms": 47.664, "count": 77, "max_ms": 66.982, "p95_ms": 64.414}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.574, "count": 77, "max_ms": 86.6, "p95_ms": 82.369}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.965, "count": 77, "max_ms": 157.971, "p95_ms": 155.061}, "read_interval": {"avg_ms": 393.397, "count": 76, "max_ms": 10136.602, "p95_ms": 168.82}, "triton_wait": {"avg_ms": 63.973, "count": 77, "max_ms": 88.421, "p95_ms": 87.086}, "visualize": {"avg_ms": 4.25, "count": 77, "max_ms": 5.485, "p95_ms": 4.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11762}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10932.625, "limit_mb": 8796093022207.996}, "detail": {"pid": 91784, "restart_count": 501, "start_count": 502}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 456, "h264_decode_error_count": 1377, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T08:59:11+09:00", "last_error_summary": "[in#0/rtsp @ 0x560ac173fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560ac173fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:59:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T08:59:14+09:00", "pid": 91784, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.59, "name": "ffmpeg", "pid": 91784, "rss_mb": 138.59, "state": "S (sleeping)", "threads": 106}, "restart_count": 501, "restart_count_per_hour": 22, "rtsp_timeout_count": 400, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 502}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.921, "sample_count": 1863, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1411463, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T08:59:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.921, "last_frame_pushed_at": "2026-07-01T08:59:11+09:00", "last_frame_read_age_sec": 5.073, "last_frame_read_at": "2026-07-01T08:59:11+09:00", "mjpeg": {"age_sec": 4.904, "exists": true, "mtime": "2026-07-01T08:59:11.937074+09:00", "mtime_epoch": 1782863951.9370742, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162617}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T08:59:16+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5038.734, "name": "celery", "pid": 267, "rss_mb": 5038.734, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.434, "pushed": 1.434, "read": 1.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.545, "count": 42, "max_ms": 33.673, "p95_ms": 32.928}, "gstreamer_push": {"avg_ms": 1.076, "count": 43, "max_ms": 1.31, "p95_ms": 1.294}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.651, "count": 42, "max_ms": 42.27, "p95_ms": 40.513}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.721, "count": 42, "max_ms": 88.409, "p95_ms": 87.074}, "yolo11_640_fire_11class": {"avg_ms": 51.321, "count": 42, "max_ms": 66.982, "p95_ms": 65.287}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.626, "count": 42, "max_ms": 86.6, "p95_ms": 84.314}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.728, "count": 43, "max_ms": 157.953, "p95_ms": 155.061}, "read_interval": {"avg_ms": 596.214, "count": 42, "max_ms": 10136.602, "p95_ms": 168.82}, "triton_wait": {"avg_ms": 67.84, "count": 42, "max_ms": 88.421, "p95_ms": 87.086}, "visualize": {"avg_ms": 4.282, "count": 42, "max_ms": 5.485, "p95_ms": 4.804}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11776}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10826.941, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x564aa27df0c0] reference picture missing during reorder [h264 @ 0x564aa27df0c0] Missing reference picture, default is 65537 [h264 @ 0x564aa27f2f80] mmco: unref short failure [h264 @ 0x564aa27f2f80] illegal short term buffer state detected [h264 @ 0x564aa27f2f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa27f2f80] error while decoding MB 0 21, bytestream 1311 [h264 @ 0x564aa28d7fc0] left block unavailable for requested intra mode [h264 @ 0x564aa28d7fc0] error wh", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 457, "h264_decode_error_count": 1382, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:03:14+09:00", "last_error_summary": "[h264 @ 0x564aa27df0c0] reference picture missing during reorder [h264 @ 0x564aa27df0c0] Missing reference picture, default is 65537 [h264 @ 0x564aa27f2f80] mmco: unref short failure [h264 @ 0x564aa27f2f80] illegal short term buffer state detected [h264 @ 0x564aa27f2f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa27f2f80] error while decoding MB 0 21, bytestream 1311 [h264 @ 0x564aa28d7fc0] left block unavailable for requested intra mode [h264 @ 0x564aa28d7fc0] error wh", "last_exit_at": "2026-07-01T08:59:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.136, "last_started_at": "2026-07-01T08:59:14+09:00", "pid": 91784, "process": {"exists": false, "pid": 91784}, "restart_count": 501, "restart_count_per_hour": 20, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 502}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.343, "sample_count": 1922, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1413127, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:03:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.109, "last_frame_pushed_at": "2026-07-01T09:03:14+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T09:03:14+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-01T09:03:14.885906+09:00", "mtime_epoch": 1782864194.8859057, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169207}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:03:14+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5039.508, "name": "celery", "pid": 267, "rss_mb": 5039.508, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 126, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.938, "pushed": 5.938, "read": 5.985}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.656, "count": 125, "max_ms": 37.321, "p95_ms": 35.723}, "gstreamer_push": {"avg_ms": 0.993, "count": 125, "max_ms": 1.099, "p95_ms": 1.066}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.474, "count": 125, "max_ms": 41.778, "p95_ms": 35.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.245, "count": 125, "max_ms": 89.226, "p95_ms": 85.428}, "yolo11_640_fire_11class": {"avg_ms": 43.758, "count": 125, "max_ms": 66.26, "p95_ms": 64.225}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.656, "count": 125, "max_ms": 86.016, "p95_ms": 83.681}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.669, "count": 125, "max_ms": 184.513, "p95_ms": 180.968}, "read_interval": {"avg_ms": 327.59, "count": 126, "max_ms": 20325.374, "p95_ms": 192.766}, "triton_wait": {"avg_ms": 63.193, "count": 125, "max_ms": 89.238, "p95_ms": 85.441}, "visualize": {"avg_ms": 5.192, "count": 125, "max_ms": 5.847, "p95_ms": 5.588}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.051, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11776}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10826.258, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:03:14+09:00", "last_error_summary": "[h264 @ 0x564aa27df0c0] reference picture missing during reorder [h264 @ 0x564aa27df0c0] Missing reference picture, default is 65537 [h264 @ 0x564aa27f2f80] mmco: unref short failure [h264 @ 0x564aa27f2f80] illegal short term buffer state detected [h264 @ 0x564aa27f2f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa27f2f80] error while decoding MB 0 21, bytestream 1311 [h264 @ 0x564aa28d7fc0] left block unavailable for requested intra mode [h264 @ 0x564aa28d7fc0] error wh", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 457, "h264_decode_error_count": 1382, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:03:14+09:00", "last_error_summary": "[h264 @ 0x564aa27df0c0] reference picture missing during reorder [h264 @ 0x564aa27df0c0] Missing reference picture, default is 65537 [h264 @ 0x564aa27f2f80] mmco: unref short failure [h264 @ 0x564aa27f2f80] illegal short term buffer state detected [h264 @ 0x564aa27f2f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa27f2f80] error while decoding MB 0 21, bytestream 1311 [h264 @ 0x564aa28d7fc0] left block unavailable for requested intra mode [h264 @ 0x564aa28d7fc0] error wh", "last_exit_at": "2026-07-01T09:03:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.194, "last_started_at": "2026-07-01T08:59:14+09:00", "pid": 91784, "process": {"exists": false, "pid": 91784}, "restart_count": 501, "restart_count_per_hour": 20, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 502}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.343, "sample_count": 1922, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1413128, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:03:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T09:03:15+09:00", "last_frame_read_age_sec": 0.157, "last_frame_read_at": "2026-07-01T09:03:14+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-01T09:03:14.885906+09:00", "mtime_epoch": 1782864194.8859057, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169207}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:03:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5039.508, "name": "celery", "pid": 267, "rss_mb": 5039.508, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.969, "pushed": 5.969, "read": 5.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.672, "count": 126, "max_ms": 37.321, "p95_ms": 35.723}, "gstreamer_push": {"avg_ms": 0.995, "count": 126, "max_ms": 1.235, "p95_ms": 1.076}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.461, "count": 126, "max_ms": 41.778, "p95_ms": 35.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.246, "count": 126, "max_ms": 89.226, "p95_ms": 85.428}, "yolo11_640_fire_11class": {"avg_ms": 43.731, "count": 126, "max_ms": 66.26, "p95_ms": 64.225}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.621, "count": 126, "max_ms": 86.016, "p95_ms": 83.681}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.621, "count": 126, "max_ms": 184.513, "p95_ms": 180.968}, "read_interval": {"avg_ms": 327.59, "count": 126, "max_ms": 20325.374, "p95_ms": 192.766}, "triton_wait": {"avg_ms": 63.187, "count": 126, "max_ms": 89.238, "p95_ms": 85.441}, "visualize": {"avg_ms": 5.209, "count": 126, "max_ms": 7.29, "p95_ms": 5.679}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.108, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11776}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10826.246, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 457, "h264_decode_error_count": 1382, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:03:14+09:00", "last_error_summary": "[h264 @ 0x564aa27df0c0] reference picture missing during reorder [h264 @ 0x564aa27df0c0] Missing reference picture, default is 65537 [h264 @ 0x564aa27f2f80] mmco: unref short failure [h264 @ 0x564aa27f2f80] illegal short term buffer state detected [h264 @ 0x564aa27f2f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa27f2f80] error while decoding MB 0 21, bytestream 1311 [h264 @ 0x564aa28d7fc0] left block unavailable for requested intra mode [h264 @ 0x564aa28d7fc0] error wh", "last_exit_at": "2026-07-01T09:03:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.2, "last_started_at": "2026-07-01T08:59:14+09:00", "pid": 91784, "process": {"exists": false, "pid": 91784}, "restart_count": 501, "restart_count_per_hour": 20, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 502}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.343, "sample_count": 1922, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1413128, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:03:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T09:03:15+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-01T09:03:14+09:00", "mjpeg": {"age_sec": 0.154, "exists": true, "mtime": "2026-07-01T09:03:14.885906+09:00", "mtime_epoch": 1782864194.8859057, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169207}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:03:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5039.508, "name": "celery", "pid": 267, "rss_mb": 5039.508, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.968, "pushed": 5.968, "read": 5.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.672, "count": 126, "max_ms": 37.321, "p95_ms": 35.723}, "gstreamer_push": {"avg_ms": 0.995, "count": 126, "max_ms": 1.235, "p95_ms": 1.076}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.461, "count": 126, "max_ms": 41.778, "p95_ms": 35.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.246, "count": 126, "max_ms": 89.226, "p95_ms": 85.428}, "yolo11_640_fire_11class": {"avg_ms": 43.731, "count": 126, "max_ms": 66.26, "p95_ms": 64.225}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.621, "count": 126, "max_ms": 86.016, "p95_ms": 83.681}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.621, "count": 126, "max_ms": 184.513, "p95_ms": 180.968}, "read_interval": {"avg_ms": 327.59, "count": 126, "max_ms": 20325.374, "p95_ms": 192.766}, "triton_wait": {"avg_ms": 63.187, "count": 126, "max_ms": 89.238, "p95_ms": 85.441}, "visualize": {"avg_ms": 5.209, "count": 126, "max_ms": 7.29, "p95_ms": 5.679}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.114, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11776}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10931.676, "limit_mb": 8796093022207.996}, "detail": {"pid": 91891, "restart_count": 502, "start_count": 503}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 457, "h264_decode_error_count": 1382, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:03:14+09:00", "last_error_summary": "[h264 @ 0x564aa27df0c0] reference picture missing during reorder [h264 @ 0x564aa27df0c0] Missing reference picture, default is 65537 [h264 @ 0x564aa27f2f80] mmco: unref short failure [h264 @ 0x564aa27f2f80] illegal short term buffer state detected [h264 @ 0x564aa27f2f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564aa27f2f80] error while decoding MB 0 21, bytestream 1311 [h264 @ 0x564aa28d7fc0] left block unavailable for requested intra mode [h264 @ 0x564aa28d7fc0] error wh", "last_exit_at": "2026-07-01T09:03:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:03:17+09:00", "pid": 91891, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.594, "name": "ffmpeg", "pid": 91891, "rss_mb": 136.594, "state": "S (sleeping)", "threads": 106}, "restart_count": 502, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 503}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.891, "sample_count": 1888, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1413128, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:03:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.713, "last_frame_pushed_at": "2026-07-01T09:03:15+09:00", "last_frame_read_age_sec": 4.866, "last_frame_read_at": "2026-07-01T09:03:14+09:00", "mjpeg": {"age_sec": 4.697, "exists": true, "mtime": "2026-07-01T09:03:15.045909+09:00", "mtime_epoch": 1782864195.045909, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170776}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:03:19+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5039.508, "name": "celery", "pid": 267, "rss_mb": 5039.508, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 5, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.88, "pushed": 4.88, "read": 4.88}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.672, "count": 126, "max_ms": 37.321, "p95_ms": 35.723}, "gstreamer_push": {"avg_ms": 0.995, "count": 126, "max_ms": 1.235, "p95_ms": 1.076}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.461, "count": 126, "max_ms": 41.778, "p95_ms": 35.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.246, "count": 126, "max_ms": 89.226, "p95_ms": 85.428}, "yolo11_640_fire_11class": {"avg_ms": 43.731, "count": 126, "max_ms": 66.26, "p95_ms": 64.225}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.621, "count": 126, "max_ms": 86.016, "p95_ms": 83.681}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.621, "count": 126, "max_ms": 184.513, "p95_ms": 180.968}, "read_interval": {"avg_ms": 327.59, "count": 126, "max_ms": 20325.374, "p95_ms": 192.766}, "triton_wait": {"avg_ms": 63.187, "count": 126, "max_ms": 89.238, "p95_ms": 85.441}, "visualize": {"avg_ms": 5.209, "count": 126, "max_ms": 7.29, "p95_ms": 5.679}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 25.817, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11782}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10827.949, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55eec3b9cbc0] error while decoding MB 45 32, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 458, "h264_decode_error_count": 1383, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:05:29+09:00", "last_error_summary": "[h264 @ 0x55eec3b9cbc0] error while decoding MB 45 32, bytestream -7", "last_exit_at": "2026-07-01T09:03:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.99, "last_started_at": "2026-07-01T09:03:17+09:00", "pid": 91891, "process": {"exists": false, "pid": 91891}, "restart_count": 502, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 503}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.792, "sample_count": 1792, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1413890, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:05:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T09:05:29+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T09:05:29+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T09:05:29.688540+09:00", "mtime_epoch": 1782864329.6885405, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162679}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:05:29+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5039.766, "name": "celery", "pid": 267, "rss_mb": 5039.766, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 175, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.84, "pushed": 5.84, "read": 5.84}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.51, "count": 174, "max_ms": 40.153, "p95_ms": 35.602}, "gstreamer_push": {"avg_ms": 0.998, "count": 175, "max_ms": 1.14, "p95_ms": 1.072}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.585, "count": 174, "max_ms": 40.543, "p95_ms": 34.288}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.201, "count": 174, "max_ms": 89.504, "p95_ms": 88.223}, "yolo11_640_fire_11class": {"avg_ms": 50.314, "count": 174, "max_ms": 69.005, "p95_ms": 66.992}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.807, "count": 174, "max_ms": 87.852, "p95_ms": 85.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 161.047, "count": 175, "max_ms": 193.838, "p95_ms": 183.217}, "read_interval": {"avg_ms": 171.849, "count": 175, "max_ms": 205.07, "p95_ms": 193.673}, "triton_wait": {"avg_ms": 65.36, "count": 174, "max_ms": 89.516, "p95_ms": 88.235}, "visualize": {"avg_ms": 5.289, "count": 174, "max_ms": 6.758, "p95_ms": 5.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11782}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10827.359, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:05:29+09:00", "last_error_summary": "[h264 @ 0x55eec3b9cbc0] error while decoding MB 45 32, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 458, "h264_decode_error_count": 1383, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:05:29+09:00", "last_error_summary": "[h264 @ 0x55eec3b9cbc0] error while decoding MB 45 32, bytestream -7", "last_exit_at": "2026-07-01T09:05:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.117, "last_started_at": "2026-07-01T09:03:17+09:00", "pid": 91891, "process": {"exists": false, "pid": 91891}, "restart_count": 502, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 503}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.792, "sample_count": 1792, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1413891, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:05:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:05:29+09:00", "last_frame_read_age_sec": 0.177, "last_frame_read_at": "2026-07-01T09:05:29+09:00", "mjpeg": {"age_sec": 0.169, "exists": true, "mtime": "2026-07-01T09:05:29.688540+09:00", "mtime_epoch": 1782864329.6885405, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162679}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:05:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5039.766, "name": "celery", "pid": 267, "rss_mb": 5039.766, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.851, "pushed": 5.851, "read": 5.818}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.495, "count": 175, "max_ms": 40.153, "p95_ms": 35.602}, "gstreamer_push": {"avg_ms": 0.998, "count": 175, "max_ms": 1.14, "p95_ms": 1.072}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.587, "count": 175, "max_ms": 40.543, "p95_ms": 34.288}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.295, "count": 175, "max_ms": 89.504, "p95_ms": 88.223}, "yolo11_640_fire_11class": {"avg_ms": 50.376, "count": 175, "max_ms": 69.005, "p95_ms": 66.992}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.903, "count": 175, "max_ms": 87.852, "p95_ms": 85.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 161.13, "count": 175, "max_ms": 193.838, "p95_ms": 183.217}, "read_interval": {"avg_ms": 171.867, "count": 174, "max_ms": 205.07, "p95_ms": 193.673}, "triton_wait": {"avg_ms": 65.454, "count": 175, "max_ms": 89.516, "p95_ms": 88.235}, "visualize": {"avg_ms": 5.285, "count": 175, "max_ms": 6.758, "p95_ms": 5.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11782}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10827.43, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 458, "h264_decode_error_count": 1383, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:05:29+09:00", "last_error_summary": "[h264 @ 0x55eec3b9cbc0] error while decoding MB 45 32, bytestream -7", "last_exit_at": "2026-07-01T09:05:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.12, "last_started_at": "2026-07-01T09:03:17+09:00", "pid": 91891, "process": {"exists": false, "pid": 91891}, "restart_count": 502, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 503}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.792, "sample_count": 1792, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1413891, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:05:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:05:29+09:00", "last_frame_read_age_sec": 0.181, "last_frame_read_at": "2026-07-01T09:05:29+09:00", "mjpeg": {"age_sec": 0.173, "exists": true, "mtime": "2026-07-01T09:05:29.688540+09:00", "mtime_epoch": 1782864329.6885405, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162679}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:05:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5039.766, "name": "celery", "pid": 267, "rss_mb": 5039.766, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.851, "pushed": 5.851, "read": 5.817}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.495, "count": 175, "max_ms": 40.153, "p95_ms": 35.602}, "gstreamer_push": {"avg_ms": 0.998, "count": 175, "max_ms": 1.14, "p95_ms": 1.072}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.587, "count": 175, "max_ms": 40.543, "p95_ms": 34.288}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.295, "count": 175, "max_ms": 89.504, "p95_ms": 88.223}, "yolo11_640_fire_11class": {"avg_ms": 50.376, "count": 175, "max_ms": 69.005, "p95_ms": 66.992}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.903, "count": 175, "max_ms": 87.852, "p95_ms": 85.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 161.13, "count": 175, "max_ms": 193.838, "p95_ms": 183.217}, "read_interval": {"avg_ms": 171.867, "count": 174, "max_ms": 205.07, "p95_ms": 193.673}, "triton_wait": {"avg_ms": 65.454, "count": 175, "max_ms": 89.516, "p95_ms": 88.235}, "visualize": {"avg_ms": 5.285, "count": 175, "max_ms": 6.758, "p95_ms": 5.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11782}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10934.82, "limit_mb": 8796093022207.996}, "detail": {"pid": 91998, "restart_count": 503, "start_count": 504}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 458, "h264_decode_error_count": 1383, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:05:29+09:00", "last_error_summary": "[h264 @ 0x55eec3b9cbc0] error while decoding MB 45 32, bytestream -7", "last_exit_at": "2026-07-01T09:05:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:05:32+09:00", "pid": 91998, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.047, "name": "ffmpeg", "pid": 91998, "rss_mb": 139.047, "state": "S (sleeping)", "threads": 106}, "restart_count": 503, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 504}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.026, "sample_count": 1751, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1413891, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:05:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.8, "last_frame_pushed_at": "2026-07-01T09:05:29+09:00", "last_frame_read_age_sec": 4.973, "last_frame_read_at": "2026-07-01T09:05:29+09:00", "mjpeg": {"age_sec": 4.786, "exists": true, "mtime": "2026-07-01T09:05:29.868544+09:00", "mtime_epoch": 1782864329.8685439, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162833}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:05:34+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5039.766, "name": "celery", "pid": 267, "rss_mb": 5039.766, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.916, "pushed": 4.916, "read": 4.883}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.384, "count": 147, "max_ms": 40.153, "p95_ms": 34.658}, "gstreamer_push": {"avg_ms": 0.993, "count": 147, "max_ms": 1.14, "p95_ms": 1.057}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.766, "count": 147, "max_ms": 36.019, "p95_ms": 34.288}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.33, "count": 147, "max_ms": 89.253, "p95_ms": 87.541}, "yolo11_640_fire_11class": {"avg_ms": 50.396, "count": 147, "max_ms": 69.005, "p95_ms": 66.284}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.027, "count": 147, "max_ms": 87.852, "p95_ms": 85.267}}, "truncated_count": 0}, "processing_total": {"avg_ms": 161.038, "count": 147, "max_ms": 193.838, "p95_ms": 182.771}, "read_interval": {"avg_ms": 171.747, "count": 146, "max_ms": 205.07, "p95_ms": 193.184}, "triton_wait": {"avg_ms": 65.516, "count": 147, "max_ms": 89.263, "p95_ms": 87.552}, "visualize": {"avg_ms": 5.281, "count": 147, "max_ms": 6.758, "p95_ms": 6.089}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11788}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10829.016, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5629ecd8e6c0] error while decoding MB 56 31, bytestream -18", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 459, "h264_decode_error_count": 1384, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:07:44+09:00", "last_error_summary": "[h264 @ 0x5629ecd8e6c0] error while decoding MB 56 31, bytestream -18", "last_exit_at": "2026-07-01T09:05:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.463, "last_started_at": "2026-07-01T09:05:32+09:00", "pid": 91998, "process": {"exists": false, "pid": 91998}, "restart_count": 503, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 504}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.991, "sample_count": 1662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1414664, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:07:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T09:07:44+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T09:07:44+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T09:07:44.075144+09:00", "mtime_epoch": 1782864464.0751436, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166098}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:07:44+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5040.281, "name": "celery", "pid": 267, "rss_mb": 5040.281, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 181, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.049, "pushed": 6.049, "read": 6.049}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.948, "count": 181, "max_ms": 36.19, "p95_ms": 34.623}, "gstreamer_push": {"avg_ms": 1.001, "count": 181, "max_ms": 1.769, "p95_ms": 1.092}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.757, "count": 181, "max_ms": 39.432, "p95_ms": 35.336}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.143, "count": 181, "max_ms": 89.271, "p95_ms": 83.289}, "yolo11_640_fire_11class": {"avg_ms": 48.096, "count": 181, "max_ms": 67.927, "p95_ms": 62.67}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.375, "count": 181, "max_ms": 88.259, "p95_ms": 82.765}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.401, "count": 181, "max_ms": 186.798, "p95_ms": 179.3}, "read_interval": {"avg_ms": 165.988, "count": 181, "max_ms": 197.41, "p95_ms": 189.374}, "triton_wait": {"avg_ms": 62.315, "count": 181, "max_ms": 89.281, "p95_ms": 85.627}, "visualize": {"avg_ms": 5.147, "count": 181, "max_ms": 6.211, "p95_ms": 5.528}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11788}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10828.469, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:07:44+09:00", "last_error_summary": "[h264 @ 0x5629ecd8e6c0] error while decoding MB 56 31, bytestream -18", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 459, "h264_decode_error_count": 1384, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:07:44+09:00", "last_error_summary": "[h264 @ 0x5629ecd8e6c0] error while decoding MB 56 31, bytestream -18", "last_exit_at": "2026-07-01T09:07:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.593, "last_started_at": "2026-07-01T09:05:32+09:00", "pid": 91998, "process": {"exists": false, "pid": 91998}, "restart_count": 503, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 504}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.991, "sample_count": 1663, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1414665, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:07:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:07:44+09:00", "last_frame_read_age_sec": 0.18, "last_frame_read_at": "2026-07-01T09:07:44+09:00", "mjpeg": {"age_sec": 0.171, "exists": true, "mtime": "2026-07-01T09:07:44.075144+09:00", "mtime_epoch": 1782864464.0751436, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166098}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:07:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5040.281, "name": "celery", "pid": 267, "rss_mb": 5040.281, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.059, "pushed": 6.059, "read": 6.026}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.93, "count": 181, "max_ms": 36.19, "p95_ms": 34.623}, "gstreamer_push": {"avg_ms": 1.002, "count": 181, "max_ms": 1.769, "p95_ms": 1.092}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.75, "count": 181, "max_ms": 39.432, "p95_ms": 35.336}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.277, "count": 181, "max_ms": 89.271, "p95_ms": 83.289}, "yolo11_640_fire_11class": {"avg_ms": 48.19, "count": 181, "max_ms": 67.927, "p95_ms": 62.67}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.482, "count": 181, "max_ms": 88.259, "p95_ms": 82.765}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.41, "count": 181, "max_ms": 186.798, "p95_ms": 179.3}, "read_interval": {"avg_ms": 165.893, "count": 180, "max_ms": 197.41, "p95_ms": 189.374}, "triton_wait": {"avg_ms": 62.436, "count": 181, "max_ms": 89.281, "p95_ms": 85.627}, "visualize": {"avg_ms": 5.141, "count": 181, "max_ms": 6.064, "p95_ms": 5.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11788}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10828.461, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 459, "h264_decode_error_count": 1384, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:07:44+09:00", "last_error_summary": "[h264 @ 0x5629ecd8e6c0] error while decoding MB 56 31, bytestream -18", "last_exit_at": "2026-07-01T09:07:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.596, "last_started_at": "2026-07-01T09:05:32+09:00", "pid": 91998, "process": {"exists": false, "pid": 91998}, "restart_count": 503, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 504}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.991, "sample_count": 1663, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1414665, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:07:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T09:07:44+09:00", "last_frame_read_age_sec": 0.184, "last_frame_read_at": "2026-07-01T09:07:44+09:00", "mjpeg": {"age_sec": 0.175, "exists": true, "mtime": "2026-07-01T09:07:44.075144+09:00", "mtime_epoch": 1782864464.0751436, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166098}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:07:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5040.281, "name": "celery", "pid": 267, "rss_mb": 5040.281, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.059, "pushed": 6.059, "read": 6.025}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.93, "count": 181, "max_ms": 36.19, "p95_ms": 34.623}, "gstreamer_push": {"avg_ms": 1.002, "count": 181, "max_ms": 1.769, "p95_ms": 1.092}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.75, "count": 181, "max_ms": 39.432, "p95_ms": 35.336}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.277, "count": 181, "max_ms": 89.271, "p95_ms": 83.289}, "yolo11_640_fire_11class": {"avg_ms": 48.19, "count": 181, "max_ms": 67.927, "p95_ms": 62.67}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.482, "count": 181, "max_ms": 88.259, "p95_ms": 82.765}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.41, "count": 181, "max_ms": 186.798, "p95_ms": 179.3}, "read_interval": {"avg_ms": 165.893, "count": 180, "max_ms": 197.41, "p95_ms": 189.374}, "triton_wait": {"avg_ms": 62.436, "count": 181, "max_ms": 89.281, "p95_ms": 85.627}, "visualize": {"avg_ms": 5.141, "count": 181, "max_ms": 6.064, "p95_ms": 5.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.875, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11788}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10936.676, "limit_mb": 8796093022207.996}, "detail": {"pid": 92105, "restart_count": 504, "start_count": 505}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 459, "h264_decode_error_count": 1384, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:07:44+09:00", "last_error_summary": "[h264 @ 0x5629ecd8e6c0] error while decoding MB 56 31, bytestream -18", "last_exit_at": "2026-07-01T09:07:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:07:46+09:00", "pid": 92105, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.129, "name": "ffmpeg", "pid": 92105, "rss_mb": 139.129, "state": "S (sleeping)", "threads": 106}, "restart_count": 504, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 505}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.991, "sample_count": 1663, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1414665, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:07:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.647, "last_frame_pushed_at": "2026-07-01T09:07:44+09:00", "last_frame_read_age_sec": 4.823, "last_frame_read_at": "2026-07-01T09:07:44+09:00", "mjpeg": {"age_sec": 4.63, "exists": true, "mtime": "2026-07-01T09:07:44.259147+09:00", "mtime_epoch": 1782864464.259147, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166124}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:07:48+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5040.281, "name": "celery", "pid": 267, "rss_mb": 5040.281, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.115, "pushed": 5.115, "read": 5.081}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.823, "count": 153, "max_ms": 36.065, "p95_ms": 34.144}, "gstreamer_push": {"avg_ms": 1.001, "count": 153, "max_ms": 1.769, "p95_ms": 1.087}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.774, "count": 153, "max_ms": 39.432, "p95_ms": 35.47}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.843, "count": 153, "max_ms": 89.271, "p95_ms": 86.992}, "yolo11_640_fire_11class": {"avg_ms": 48.449, "count": 153, "max_ms": 67.927, "p95_ms": 64.613}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.088, "count": 153, "max_ms": 88.259, "p95_ms": 83.312}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.637, "count": 153, "max_ms": 186.798, "p95_ms": 179.494}, "read_interval": {"avg_ms": 166.084, "count": 152, "max_ms": 197.41, "p95_ms": 189.701}, "triton_wait": {"avg_ms": 62.992, "count": 153, "max_ms": 89.281, "p95_ms": 87.005}, "visualize": {"avg_ms": 5.144, "count": 153, "max_ms": 5.998, "p95_ms": 5.521}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11795}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10830.281, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b8e55ed680] error while decoding MB 60 9, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 460, "h264_decode_error_count": 1385, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:09:59+09:00", "last_error_summary": "[h264 @ 0x55b8e55ed680] error while decoding MB 60 9, bytestream -7", "last_exit_at": "2026-07-01T09:07:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.562, "last_started_at": "2026-07-01T09:07:46+09:00", "pid": 92105, "process": {"exists": false, "pid": 92105}, "restart_count": 504, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 505}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.481, "sample_count": 1729, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1415443, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:09:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T09:09:59+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T09:09:59+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T09:09:59.409747+09:00", "mtime_epoch": 1782864599.409747, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162755}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:09:59+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5040.539, "name": "celery", "pid": 267, "rss_mb": 5040.539, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 176, "pushed_frames": 176, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.897, "pushed": 5.897, "read": 5.897}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.752, "count": 176, "max_ms": 36.244, "p95_ms": 35.039}, "gstreamer_push": {"avg_ms": 1.001, "count": 176, "max_ms": 1.55, "p95_ms": 1.098}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.974, "count": 176, "max_ms": 38.315, "p95_ms": 34.508}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.2, "count": 176, "max_ms": 90.138, "p95_ms": 83.763}, "yolo11_640_fire_11class": {"avg_ms": 48.299, "count": 176, "max_ms": 68.972, "p95_ms": 61.302}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.986, "count": 176, "max_ms": 85.729, "p95_ms": 80.211}}, "truncated_count": 0}, "processing_total": {"avg_ms": 159.571, "count": 176, "max_ms": 198.931, "p95_ms": 181.393}, "read_interval": {"avg_ms": 170.343, "count": 176, "max_ms": 208.792, "p95_ms": 192.142}, "triton_wait": {"avg_ms": 63.404, "count": 176, "max_ms": 90.15, "p95_ms": 83.776}, "visualize": {"avg_ms": 5.284, "count": 176, "max_ms": 6.477, "p95_ms": 5.975}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11795}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10829.727, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:09:59+09:00", "last_error_summary": "[h264 @ 0x55b8e55ed680] error while decoding MB 60 9, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 460, "h264_decode_error_count": 1385, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:09:59+09:00", "last_error_summary": "[h264 @ 0x55b8e55ed680] error while decoding MB 60 9, bytestream -7", "last_exit_at": "2026-07-01T09:09:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.672, "last_started_at": "2026-07-01T09:07:46+09:00", "pid": 92105, "process": {"exists": false, "pid": 92105}, "restart_count": 504, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 505}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.481, "sample_count": 1730, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1415444, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:09:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:09:59+09:00", "last_frame_read_age_sec": 0.16, "last_frame_read_at": "2026-07-01T09:09:59+09:00", "mjpeg": {"age_sec": 0.151, "exists": true, "mtime": "2026-07-01T09:09:59.409747+09:00", "mtime_epoch": 1782864599.409747, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162755}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:09:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5040.539, "name": "celery", "pid": 267, "rss_mb": 5040.539, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.909, "pushed": 5.909, "read": 5.875}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.748, "count": 176, "max_ms": 36.244, "p95_ms": 35.039}, "gstreamer_push": {"avg_ms": 1.001, "count": 177, "max_ms": 1.55, "p95_ms": 1.098}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.978, "count": 176, "max_ms": 38.315, "p95_ms": 34.508}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.076, "count": 176, "max_ms": 90.138, "p95_ms": 83.282}, "yolo11_640_fire_11class": {"avg_ms": 48.307, "count": 176, "max_ms": 68.972, "p95_ms": 61.302}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.868, "count": 176, "max_ms": 85.729, "p95_ms": 80.211}}, "truncated_count": 0}, "processing_total": {"avg_ms": 159.543, "count": 177, "max_ms": 198.931, "p95_ms": 181.393}, "read_interval": {"avg_ms": 170.343, "count": 176, "max_ms": 208.792, "p95_ms": 192.142}, "triton_wait": {"avg_ms": 63.279, "count": 176, "max_ms": 90.15, "p95_ms": 83.295}, "visualize": {"avg_ms": 5.282, "count": 176, "max_ms": 6.477, "p95_ms": 5.975}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11795}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10829.84, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 460, "h264_decode_error_count": 1385, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:09:59+09:00", "last_error_summary": "[h264 @ 0x55b8e55ed680] error while decoding MB 60 9, bytestream -7", "last_exit_at": "2026-07-01T09:09:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.677, "last_started_at": "2026-07-01T09:07:46+09:00", "pid": 92105, "process": {"exists": false, "pid": 92105}, "restart_count": 504, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 505}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.481, "sample_count": 1730, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1415444, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:09:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:09:59+09:00", "last_frame_read_age_sec": 0.164, "last_frame_read_at": "2026-07-01T09:09:59+09:00", "mjpeg": {"age_sec": 0.155, "exists": true, "mtime": "2026-07-01T09:09:59.409747+09:00", "mtime_epoch": 1782864599.409747, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162755}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:09:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5040.539, "name": "celery", "pid": 267, "rss_mb": 5040.539, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.908, "pushed": 5.908, "read": 5.874}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.748, "count": 176, "max_ms": 36.244, "p95_ms": 35.039}, "gstreamer_push": {"avg_ms": 1.001, "count": 177, "max_ms": 1.55, "p95_ms": 1.098}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.978, "count": 176, "max_ms": 38.315, "p95_ms": 34.508}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.076, "count": 176, "max_ms": 90.138, "p95_ms": 83.282}, "yolo11_640_fire_11class": {"avg_ms": 48.307, "count": 176, "max_ms": 68.972, "p95_ms": 61.302}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.868, "count": 176, "max_ms": 85.729, "p95_ms": 80.211}}, "truncated_count": 0}, "processing_total": {"avg_ms": 159.543, "count": 177, "max_ms": 198.931, "p95_ms": 181.393}, "read_interval": {"avg_ms": 170.343, "count": 176, "max_ms": 208.792, "p95_ms": 192.142}, "triton_wait": {"avg_ms": 63.279, "count": 176, "max_ms": 90.15, "p95_ms": 83.295}, "visualize": {"avg_ms": 5.282, "count": 176, "max_ms": 6.477, "p95_ms": 5.975}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11795}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10937.812, "limit_mb": 8796093022207.996}, "detail": {"pid": 92212, "restart_count": 505, "start_count": 506}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 460, "h264_decode_error_count": 1385, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:09:59+09:00", "last_error_summary": "[h264 @ 0x55b8e55ed680] error while decoding MB 60 9, bytestream -7", "last_exit_at": "2026-07-01T09:10:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:10:01+09:00", "pid": 92212, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.25, "name": "ffmpeg", "pid": 92212, "rss_mb": 139.25, "state": "S (sleeping)", "threads": 106}, "restart_count": 505, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 506}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.161, "sample_count": 1704, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1415444, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:09:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.402, "last_frame_pushed_at": "2026-07-01T09:09:59+09:00", "last_frame_read_age_sec": 4.558, "last_frame_read_at": "2026-07-01T09:09:59+09:00", "mjpeg": {"age_sec": 4.385, "exists": true, "mtime": "2026-07-01T09:09:59.573750+09:00", "mtime_epoch": 1782864599.57375, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162696}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:10:03+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5040.539, "name": "celery", "pid": 267, "rss_mb": 5040.539, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 151, "pushed_frames": 151, "read_frames": 150, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.042, "pushed": 5.042, "read": 5.008}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.682, "count": 150, "max_ms": 36.244, "p95_ms": 34.814}, "gstreamer_push": {"avg_ms": 0.998, "count": 151, "max_ms": 1.142, "p95_ms": 1.098}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.854, "count": 150, "max_ms": 38.315, "p95_ms": 34.508}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.217, "count": 150, "max_ms": 90.138, "p95_ms": 83.282}, "yolo11_640_fire_11class": {"avg_ms": 48.288, "count": 150, "max_ms": 68.972, "p95_ms": 61.03}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.768, "count": 150, "max_ms": 85.729, "p95_ms": 80.211}}, "truncated_count": 0}, "processing_total": {"avg_ms": 159.552, "count": 151, "max_ms": 198.931, "p95_ms": 181.393}, "read_interval": {"avg_ms": 170.293, "count": 150, "max_ms": 208.792, "p95_ms": 192.142}, "triton_wait": {"avg_ms": 63.389, "count": 150, "max_ms": 90.15, "p95_ms": 83.295}, "visualize": {"avg_ms": 5.305, "count": 150, "max_ms": 6.477, "p95_ms": 6.06}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10831.277, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e8e30d8580] left block unavailable for requested intra mode [h264 @ 0x55e8e30d8580] error while decoding MB 0 25, bytestream 7242 [h264 @ 0x55e8e30fad00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e8e30fad00] error while decoding MB 0 29, bytestream 3759 [h264 @ 0x55e8e3cae900] error while decoding MB 46 19, bytestream -8", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 461, "h264_decode_error_count": 1388, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:12:35+09:00", "last_error_summary": "[h264 @ 0x55e8e30d8580] left block unavailable for requested intra mode [h264 @ 0x55e8e30d8580] error while decoding MB 0 25, bytestream 7242 [h264 @ 0x55e8e30fad00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e8e30fad00] error while decoding MB 0 29, bytestream 3759 [h264 @ 0x55e8e3cae900] error while decoding MB 46 19, bytestream -8", "last_exit_at": "2026-07-01T09:10:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.163, "last_started_at": "2026-07-01T09:10:01+09:00", "pid": 92212, "process": {"exists": false, "pid": 92212}, "restart_count": 505, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 506}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.496, "sample_count": 1746, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1416354, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:12:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-01T09:12:35+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-01T09:12:35+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-01T09:12:35.068724+09:00", "mtime_epoch": 1782864755.068724, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163865}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:12:35+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5041.055, "name": "celery", "pid": 267, "rss_mb": 5041.055, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 178, "pushed_frames": 178, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.943, "pushed": 5.943, "read": 5.943}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.985, "count": 177, "max_ms": 38.381, "p95_ms": 36.678}, "gstreamer_push": {"avg_ms": 1.0, "count": 178, "max_ms": 1.443, "p95_ms": 1.06}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.381, "count": 177, "max_ms": 40.825, "p95_ms": 36.158}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.484, "count": 177, "max_ms": 89.804, "p95_ms": 83.44}, "yolo11_640_fire_11class": {"avg_ms": 48.774, "count": 177, "max_ms": 71.074, "p95_ms": 64.283}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.256, "count": 177, "max_ms": 87.853, "p95_ms": 81.846}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.007, "count": 178, "max_ms": 186.831, "p95_ms": 180.084}, "read_interval": {"avg_ms": 168.821, "count": 178, "max_ms": 197.349, "p95_ms": 190.633}, "triton_wait": {"avg_ms": 62.231, "count": 177, "max_ms": 89.817, "p95_ms": 84.005}, "visualize": {"avg_ms": 5.22, "count": 177, "max_ms": 6.526, "p95_ms": 5.578}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10830.695, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:12:35+09:00", "last_error_summary": "[h264 @ 0x55e8e30d8580] left block unavailable for requested intra mode [h264 @ 0x55e8e30d8580] error while decoding MB 0 25, bytestream 7242 [h264 @ 0x55e8e30fad00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e8e30fad00] error while decoding MB 0 29, bytestream 3759 [h264 @ 0x55e8e3cae900] error while decoding MB 46 19, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 461, "h264_decode_error_count": 1388, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:12:35+09:00", "last_error_summary": "[h264 @ 0x55e8e30d8580] left block unavailable for requested intra mode [h264 @ 0x55e8e30d8580] error while decoding MB 0 25, bytestream 7242 [h264 @ 0x55e8e30fad00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e8e30fad00] error while decoding MB 0 29, bytestream 3759 [h264 @ 0x55e8e3cae900] error while decoding MB 46 19, bytestream -8", "last_exit_at": "2026-07-01T09:12:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.273, "last_started_at": "2026-07-01T09:10:01+09:00", "pid": 92212, "process": {"exists": false, "pid": 92212}, "restart_count": 505, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 506}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.496, "sample_count": 1746, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1416355, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:12:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:12:35+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T09:12:35+09:00", "mjpeg": {"age_sec": 0.162, "exists": true, "mtime": "2026-07-01T09:12:35.068724+09:00", "mtime_epoch": 1782864755.068724, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163865}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:12:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5041.055, "name": "celery", "pid": 267, "rss_mb": 5041.055, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 178, "pushed_frames": 178, "read_frames": 177, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.953, "pushed": 5.953, "read": 5.92}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.004, "count": 178, "max_ms": 38.381, "p95_ms": 36.678}, "gstreamer_push": {"avg_ms": 1.0, "count": 178, "max_ms": 1.443, "p95_ms": 1.06}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.374, "count": 178, "max_ms": 40.825, "p95_ms": 36.158}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.498, "count": 178, "max_ms": 89.804, "p95_ms": 83.44}, "yolo11_640_fire_11class": {"avg_ms": 48.809, "count": 178, "max_ms": 71.074, "p95_ms": 64.283}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.266, "count": 178, "max_ms": 87.853, "p95_ms": 81.846}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.055, "count": 178, "max_ms": 186.831, "p95_ms": 180.084}, "read_interval": {"avg_ms": 168.847, "count": 177, "max_ms": 197.349, "p95_ms": 190.633}, "triton_wait": {"avg_ms": 62.242, "count": 178, "max_ms": 89.817, "p95_ms": 84.005}, "visualize": {"avg_ms": 5.219, "count": 178, "max_ms": 6.526, "p95_ms": 5.578}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10830.688, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 461, "h264_decode_error_count": 1388, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:12:35+09:00", "last_error_summary": "[h264 @ 0x55e8e30d8580] left block unavailable for requested intra mode [h264 @ 0x55e8e30d8580] error while decoding MB 0 25, bytestream 7242 [h264 @ 0x55e8e30fad00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e8e30fad00] error while decoding MB 0 29, bytestream 3759 [h264 @ 0x55e8e3cae900] error while decoding MB 46 19, bytestream -8", "last_exit_at": "2026-07-01T09:12:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.277, "last_started_at": "2026-07-01T09:10:01+09:00", "pid": 92212, "process": {"exists": false, "pid": 92212}, "restart_count": 505, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 506}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.496, "sample_count": 1746, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1416355, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:12:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:12:35+09:00", "last_frame_read_age_sec": 0.172, "last_frame_read_at": "2026-07-01T09:12:35+09:00", "mjpeg": {"age_sec": 0.166, "exists": true, "mtime": "2026-07-01T09:12:35.068724+09:00", "mtime_epoch": 1782864755.068724, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163865}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:12:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5041.055, "name": "celery", "pid": 267, "rss_mb": 5041.055, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 178, "pushed_frames": 178, "read_frames": 177, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.952, "pushed": 5.952, "read": 5.919}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.004, "count": 178, "max_ms": 38.381, "p95_ms": 36.678}, "gstreamer_push": {"avg_ms": 1.0, "count": 178, "max_ms": 1.443, "p95_ms": 1.06}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.374, "count": 178, "max_ms": 40.825, "p95_ms": 36.158}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.498, "count": 178, "max_ms": 89.804, "p95_ms": 83.44}, "yolo11_640_fire_11class": {"avg_ms": 48.809, "count": 178, "max_ms": 71.074, "p95_ms": 64.283}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.266, "count": 178, "max_ms": 87.853, "p95_ms": 81.846}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.055, "count": 178, "max_ms": 186.831, "p95_ms": 180.084}, "read_interval": {"avg_ms": 168.847, "count": 177, "max_ms": 197.349, "p95_ms": 190.633}, "triton_wait": {"avg_ms": 62.242, "count": 178, "max_ms": 89.817, "p95_ms": 84.005}, "visualize": {"avg_ms": 5.219, "count": 178, "max_ms": 6.526, "p95_ms": 5.578}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10936.266, "limit_mb": 8796093022207.996}, "detail": {"pid": 92319, "restart_count": 506, "start_count": 507}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 461, "h264_decode_error_count": 1388, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:12:35+09:00", "last_error_summary": "[h264 @ 0x55e8e30d8580] left block unavailable for requested intra mode [h264 @ 0x55e8e30d8580] error while decoding MB 0 25, bytestream 7242 [h264 @ 0x55e8e30fad00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e8e30fad00] error while decoding MB 0 29, bytestream 3759 [h264 @ 0x55e8e3cae900] error while decoding MB 46 19, bytestream -8", "last_exit_at": "2026-07-01T09:12:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:12:37+09:00", "pid": 92319, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.605, "name": "ffmpeg", "pid": 92319, "rss_mb": 136.605, "state": "S (sleeping)", "threads": 106}, "restart_count": 506, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 507}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.62, "sample_count": 1721, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1416355, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:12:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.022, "last_frame_pushed_at": "2026-07-01T09:12:35+09:00", "last_frame_read_age_sec": 4.186, "last_frame_read_at": "2026-07-01T09:12:35+09:00", "mjpeg": {"age_sec": 4.005, "exists": true, "mtime": "2026-07-01T09:12:35.244727+09:00", "mtime_epoch": 1782864755.2447274, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165027}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:12:39+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5041.055, "name": "celery", "pid": 267, "rss_mb": 5041.055, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 154, "pushed_frames": 154, "read_frames": 153, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.152, "pushed": 5.152, "read": 5.119}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.003, "count": 154, "max_ms": 38.381, "p95_ms": 37.107}, "gstreamer_push": {"avg_ms": 1.005, "count": 154, "max_ms": 1.443, "p95_ms": 1.07}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.247, "count": 154, "max_ms": 40.825, "p95_ms": 36.158}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.44, "count": 154, "max_ms": 89.804, "p95_ms": 83.207}, "yolo11_640_fire_11class": {"avg_ms": 48.928, "count": 154, "max_ms": 71.074, "p95_ms": 64.129}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.176, "count": 154, "max_ms": 87.853, "p95_ms": 80.98}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.234, "count": 154, "max_ms": 186.831, "p95_ms": 179.823}, "read_interval": {"avg_ms": 169.033, "count": 153, "max_ms": 197.349, "p95_ms": 189.906}, "triton_wait": {"avg_ms": 62.298, "count": 154, "max_ms": 89.817, "p95_ms": 83.452}, "visualize": {"avg_ms": 5.223, "count": 154, "max_ms": 6.526, "p95_ms": 5.585}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11809}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10832.68, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a662a6ce40] left block unavailable for requested intra mode [h264 @ 0x55a662a6ce40] error while decoding MB 0 7, bytestream 3185 [h264 @ 0x55a662a6ce40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a662a6ce40] error while decoding MB 0 2, bytestream 2727 [h264 @ 0x55a662cd99c0] error while decoding MB 28 31, bytestream -21", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 462, "h264_decode_error_count": 1391, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:14:56+09:00", "last_error_summary": "[h264 @ 0x55a662a6ce40] left block unavailable for requested intra mode [h264 @ 0x55a662a6ce40] error while decoding MB 0 7, bytestream 3185 [h264 @ 0x55a662a6ce40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a662a6ce40] error while decoding MB 0 2, bytestream 2727 [h264 @ 0x55a662cd99c0] error while decoding MB 28 31, bytestream -21", "last_exit_at": "2026-07-01T09:12:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.478, "last_started_at": "2026-07-01T09:12:37+09:00", "pid": 92319, "process": {"exists": false, "pid": 92319}, "restart_count": 506, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 507}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 184.239, "sample_count": 1660, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1417087, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:14:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-01T09:14:56+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-01T09:14:56+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-01T09:14:56.670430+09:00", "mtime_epoch": 1782864896.67043, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168013}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:14:56+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5041.312, "name": "celery", "pid": 267, "rss_mb": 5041.312, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 162, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.422, "pushed": 5.422, "read": 5.422}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget"], "latency_ms": {"analyze": {"avg_ms": 34.2, "count": 162, "max_ms": 39.847, "p95_ms": 36.614}, "gstreamer_push": {"avg_ms": 0.897, "count": 162, "max_ms": 1.426, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.418, "count": 162, "max_ms": 37.494, "p95_ms": 34.584}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.035, "count": 162, "max_ms": 77.892, "p95_ms": 65.982}, "yolo11_640_fire_11class": {"avg_ms": 46.134, "count": 162, "max_ms": 61.599, "p95_ms": 54.337}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.536, "count": 162, "max_ms": 82.264, "p95_ms": 68.593}}, "truncated_count": 0}, "processing_total": {"avg_ms": 174.681, "count": 162, "max_ms": 197.523, "p95_ms": 185.696}, "read_interval": {"avg_ms": 184.977, "count": 162, "max_ms": 207.356, "p95_ms": 196.478}, "triton_wait": {"avg_ms": 58.091, "count": 162, "max_ms": 82.277, "p95_ms": 68.605}, "visualize": {"avg_ms": 6.115, "count": 162, "max_ms": 7.245, "p95_ms": 6.699}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11809}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10832.414, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:14:56+09:00", "last_error_summary": "[h264 @ 0x55a662a6ce40] left block unavailable for requested intra mode [h264 @ 0x55a662a6ce40] error while decoding MB 0 7, bytestream 3185 [h264 @ 0x55a662a6ce40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a662a6ce40] error while decoding MB 0 2, bytestream 2727 [h264 @ 0x55a662cd99c0] error while decoding MB 28 31, bytestream -21", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 462, "h264_decode_error_count": 1391, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:14:56+09:00", "last_error_summary": "[h264 @ 0x55a662a6ce40] left block unavailable for requested intra mode [h264 @ 0x55a662a6ce40] error while decoding MB 0 7, bytestream 3185 [h264 @ 0x55a662a6ce40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a662a6ce40] error while decoding MB 0 2, bytestream 2727 [h264 @ 0x55a662cd99c0] error while decoding MB 28 31, bytestream -21", "last_exit_at": "2026-07-01T09:14:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.61, "last_started_at": "2026-07-01T09:12:37+09:00", "pid": 92319, "process": {"exists": false, "pid": 92319}, "restart_count": 506, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 507}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 184.27, "sample_count": 1660, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1417088, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:14:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T09:14:56+09:00", "last_frame_read_age_sec": 0.195, "last_frame_read_at": "2026-07-01T09:14:56+09:00", "mjpeg": {"age_sec": 0.188, "exists": true, "mtime": "2026-07-01T09:14:56.670430+09:00", "mtime_epoch": 1782864896.67043, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168013}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:14:56+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5041.312, "name": "celery", "pid": 267, "rss_mb": 5041.312, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 163, "pushed_frames": 163, "read_frames": 162, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.436, "pushed": 5.436, "read": 5.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget"], "latency_ms": {"analyze": {"avg_ms": 34.212, "count": 162, "max_ms": 39.847, "p95_ms": 36.614}, "gstreamer_push": {"avg_ms": 0.897, "count": 163, "max_ms": 1.426, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.399, "count": 162, "max_ms": 37.494, "p95_ms": 34.584}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.026, "count": 162, "max_ms": 77.892, "p95_ms": 65.982}, "yolo11_640_fire_11class": {"avg_ms": 46.138, "count": 162, "max_ms": 61.599, "p95_ms": 54.337}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.548, "count": 162, "max_ms": 82.264, "p95_ms": 68.593}}, "truncated_count": 0}, "processing_total": {"avg_ms": 174.777, "count": 163, "max_ms": 197.523, "p95_ms": 187.112}, "read_interval": {"avg_ms": 184.977, "count": 162, "max_ms": 207.356, "p95_ms": 196.478}, "triton_wait": {"avg_ms": 58.093, "count": 162, "max_ms": 82.277, "p95_ms": 68.605}, "visualize": {"avg_ms": 6.135, "count": 162, "max_ms": 9.062, "p95_ms": 6.796}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11809}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10832.531, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 462, "h264_decode_error_count": 1391, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:14:56+09:00", "last_error_summary": "[h264 @ 0x55a662a6ce40] left block unavailable for requested intra mode [h264 @ 0x55a662a6ce40] error while decoding MB 0 7, bytestream 3185 [h264 @ 0x55a662a6ce40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a662a6ce40] error while decoding MB 0 2, bytestream 2727 [h264 @ 0x55a662cd99c0] error while decoding MB 28 31, bytestream -21", "last_exit_at": "2026-07-01T09:14:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.614, "last_started_at": "2026-07-01T09:12:37+09:00", "pid": 92319, "process": {"exists": false, "pid": 92319}, "restart_count": 506, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 507}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 184.27, "sample_count": 1660, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1417088, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:14:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T09:14:56+09:00", "last_frame_read_age_sec": 0.199, "last_frame_read_at": "2026-07-01T09:14:56+09:00", "mjpeg": {"age_sec": 0.192, "exists": true, "mtime": "2026-07-01T09:14:56.670430+09:00", "mtime_epoch": 1782864896.67043, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168013}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:14:56+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5041.312, "name": "celery", "pid": 267, "rss_mb": 5041.312, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 163, "pushed_frames": 163, "read_frames": 162, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.435, "pushed": 5.435, "read": 5.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget"], "latency_ms": {"analyze": {"avg_ms": 34.212, "count": 162, "max_ms": 39.847, "p95_ms": 36.614}, "gstreamer_push": {"avg_ms": 0.897, "count": 163, "max_ms": 1.426, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.399, "count": 162, "max_ms": 37.494, "p95_ms": 34.584}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.026, "count": 162, "max_ms": 77.892, "p95_ms": 65.982}, "yolo11_640_fire_11class": {"avg_ms": 46.138, "count": 162, "max_ms": 61.599, "p95_ms": 54.337}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.548, "count": 162, "max_ms": 82.264, "p95_ms": 68.593}}, "truncated_count": 0}, "processing_total": {"avg_ms": 174.777, "count": 163, "max_ms": 197.523, "p95_ms": 187.112}, "read_interval": {"avg_ms": 184.977, "count": 162, "max_ms": 207.356, "p95_ms": 196.478}, "triton_wait": {"avg_ms": 58.093, "count": 162, "max_ms": 82.277, "p95_ms": 68.605}, "visualize": {"avg_ms": 6.135, "count": 162, "max_ms": 9.062, "p95_ms": 6.796}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11809}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10938.344, "limit_mb": 8796093022207.996}, "detail": {"pid": 92426, "restart_count": 507, "start_count": 508}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 462, "h264_decode_error_count": 1391, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:14:56+09:00", "last_error_summary": "[h264 @ 0x55a662a6ce40] left block unavailable for requested intra mode [h264 @ 0x55a662a6ce40] error while decoding MB 0 7, bytestream 3185 [h264 @ 0x55a662a6ce40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a662a6ce40] error while decoding MB 0 2, bytestream 2727 [h264 @ 0x55a662cd99c0] error while decoding MB 28 31, bytestream -21", "last_exit_at": "2026-07-01T09:14:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:14:59+09:00", "pid": 92426, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.422, "name": "ffmpeg", "pid": 92426, "rss_mb": 136.422, "state": "S (sleeping)", "threads": 106}, "restart_count": 507, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 508}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 184.297, "sample_count": 1644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1417088, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:14:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.246, "last_frame_pushed_at": "2026-07-01T09:14:56+09:00", "last_frame_read_age_sec": 4.438, "last_frame_read_at": "2026-07-01T09:14:56+09:00", "mjpeg": {"age_sec": 4.231, "exists": true, "mtime": "2026-07-01T09:14:56.870431+09:00", "mtime_epoch": 1782864896.8704312, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165157}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:15:01+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5041.312, "name": "celery", "pid": 267, "rss_mb": 5041.312, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.67, "pushed": 4.67, "read": 4.637}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget"], "latency_ms": {"analyze": {"avg_ms": 34.228, "count": 139, "max_ms": 38.263, "p95_ms": 36.681}, "gstreamer_push": {"avg_ms": 0.894, "count": 140, "max_ms": 1.018, "p95_ms": 0.941}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 28.907, "count": 139, "max_ms": 37.494, "p95_ms": 33.725}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.807, "count": 139, "max_ms": 77.892, "p95_ms": 65.982}, "yolo11_640_fire_11class": {"avg_ms": 45.919, "count": 139, "max_ms": 59.414, "p95_ms": 53.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.435, "count": 139, "max_ms": 82.264, "p95_ms": 68.593}}, "truncated_count": 0}, "processing_total": {"avg_ms": 174.774, "count": 140, "max_ms": 197.523, "p95_ms": 187.112}, "read_interval": {"avg_ms": 184.939, "count": 139, "max_ms": 207.356, "p95_ms": 197.713}, "triton_wait": {"avg_ms": 58.002, "count": 139, "max_ms": 82.277, "p95_ms": 68.605}, "visualize": {"avg_ms": 6.151, "count": 139, "max_ms": 9.062, "p95_ms": 6.805}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11815}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10833.93, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:14:56+09:00", "last_error_summary": "[h264 @ 0x55a662a6ce40] left block unavailable for requested intra mode [h264 @ 0x55a662a6ce40] error while decoding MB 0 7, bytestream 3185 [h264 @ 0x55a662a6ce40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a662a6ce40] error while decoding MB 0 2, bytestream 2727 [h264 @ 0x55a662cd99c0] error while decoding MB 28 31, bytestream -21", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 462, "h264_decode_error_count": 1391, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:14:56+09:00", "last_error_summary": "[h264 @ 0x55a662a6ce40] left block unavailable for requested intra mode [h264 @ 0x55a662a6ce40] error while decoding MB 0 7, bytestream 3185 [h264 @ 0x55a662a6ce40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a662a6ce40] error while decoding MB 0 2, bytestream 2727 [h264 @ 0x55a662cd99c0] error while decoding MB 28 31, bytestream -21", "last_exit_at": "2026-07-01T09:17:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.392, "last_started_at": "2026-07-01T09:14:59+09:00", "pid": 92426, "process": {"exists": false, "pid": 92426}, "restart_count": 507, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 508}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 191.42, "sample_count": 1609, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1417805, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:17:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:17:08+09:00", "last_frame_read_age_sec": 0.162, "last_frame_read_at": "2026-07-01T09:17:08+09:00", "mjpeg": {"age_sec": 0.157, "exists": true, "mtime": "2026-07-01T09:17:08.335419+09:00", "mtime_epoch": 1782865028.3354194, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:17:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5041.828, "name": "celery", "pid": 267, "rss_mb": 5041.828, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.043, "pushed": 6.043, "read": 6.01}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.465, "count": 180, "max_ms": 43.95, "p95_ms": 39.926}, "gstreamer_push": {"avg_ms": 0.988, "count": 181, "max_ms": 1.547, "p95_ms": 1.108}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.534, "count": 180, "max_ms": 47.163, "p95_ms": 37.029}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.968, "count": 180, "max_ms": 81.421, "p95_ms": 74.685}, "yolo11_640_fire_11class": {"avg_ms": 44.967, "count": 180, "max_ms": 64.981, "p95_ms": 61.591}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.666, "count": 180, "max_ms": 85.389, "p95_ms": 80.994}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.785, "count": 181, "max_ms": 195.127, "p95_ms": 178.114}, "read_interval": {"avg_ms": 166.455, "count": 180, "max_ms": 205.368, "p95_ms": 188.318}, "triton_wait": {"avg_ms": 58.707, "count": 180, "max_ms": 85.401, "p95_ms": 81.089}, "visualize": {"avg_ms": 5.21, "count": 180, "max_ms": 6.543, "p95_ms": 6.119}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11815}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10834.09, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 462, "h264_decode_error_count": 1391, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:14:56+09:00", "last_error_summary": "[h264 @ 0x55a662a6ce40] left block unavailable for requested intra mode [h264 @ 0x55a662a6ce40] error while decoding MB 0 7, bytestream 3185 [h264 @ 0x55a662a6ce40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a662a6ce40] error while decoding MB 0 2, bytestream 2727 [h264 @ 0x55a662cd99c0] error while decoding MB 28 31, bytestream -21", "last_exit_at": "2026-07-01T09:17:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.396, "last_started_at": "2026-07-01T09:14:59+09:00", "pid": 92426, "process": {"exists": false, "pid": 92426}, "restart_count": 507, "restart_count_per_hour": 21, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 508}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 191.42, "sample_count": 1609, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1417805, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:17:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:17:08+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T09:17:08+09:00", "mjpeg": {"age_sec": 0.161, "exists": true, "mtime": "2026-07-01T09:17:08.335419+09:00", "mtime_epoch": 1782865028.3354194, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:17:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5041.828, "name": "celery", "pid": 267, "rss_mb": 5041.828, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.042, "pushed": 6.042, "read": 6.009}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.465, "count": 180, "max_ms": 43.95, "p95_ms": 39.926}, "gstreamer_push": {"avg_ms": 0.988, "count": 181, "max_ms": 1.547, "p95_ms": 1.108}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.534, "count": 180, "max_ms": 47.163, "p95_ms": 37.029}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.968, "count": 180, "max_ms": 81.421, "p95_ms": 74.685}, "yolo11_640_fire_11class": {"avg_ms": 44.967, "count": 180, "max_ms": 64.981, "p95_ms": 61.591}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.666, "count": 180, "max_ms": 85.389, "p95_ms": 80.994}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.785, "count": 181, "max_ms": 195.127, "p95_ms": 178.114}, "read_interval": {"avg_ms": 166.455, "count": 180, "max_ms": 205.368, "p95_ms": 188.318}, "triton_wait": {"avg_ms": 58.707, "count": 180, "max_ms": 85.401, "p95_ms": 81.089}, "visualize": {"avg_ms": 5.21, "count": 180, "max_ms": 6.543, "p95_ms": 6.119}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11815}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10943.027, "limit_mb": 8796093022207.996}, "detail": {"pid": 92533, "restart_count": 508, "start_count": 509}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 462, "h264_decode_error_count": 1391, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:14:56+09:00", "last_error_summary": "[h264 @ 0x55a662a6ce40] left block unavailable for requested intra mode [h264 @ 0x55a662a6ce40] error while decoding MB 0 7, bytestream 3185 [h264 @ 0x55a662a6ce40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a662a6ce40] error while decoding MB 0 2, bytestream 2727 [h264 @ 0x55a662cd99c0] error while decoding MB 28 31, bytestream -21", "last_exit_at": "2026-07-01T09:17:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:17:11+09:00", "pid": 92533, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.418, "name": "ffmpeg", "pid": 92533, "rss_mb": 139.418, "state": "S (sleeping)", "threads": 106}, "restart_count": 508, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 509}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 191.637, "sample_count": 1581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1417805, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:17:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.717, "last_frame_pushed_at": "2026-07-01T09:17:08+09:00", "last_frame_read_age_sec": 4.876, "last_frame_read_at": "2026-07-01T09:17:08+09:00", "mjpeg": {"age_sec": 4.698, "exists": true, "mtime": "2026-07-01T09:17:08.507421+09:00", "mtime_epoch": 1782865028.5074208, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163444}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:17:13+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5041.828, "name": "celery", "pid": 267, "rss_mb": 5041.828, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.112, "pushed": 5.112, "read": 5.079}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.371, "count": 153, "max_ms": 43.95, "p95_ms": 41.24}, "gstreamer_push": {"avg_ms": 0.984, "count": 153, "max_ms": 1.182, "p95_ms": 1.111}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.49, "count": 153, "max_ms": 47.163, "p95_ms": 37.327}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.379, "count": 153, "max_ms": 81.421, "p95_ms": 74.638}, "yolo11_640_fire_11class": {"avg_ms": 45.049, "count": 153, "max_ms": 64.981, "p95_ms": 62.372}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.524, "count": 153, "max_ms": 85.389, "p95_ms": 81.077}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.303, "count": 153, "max_ms": 195.127, "p95_ms": 179.308}, "read_interval": {"avg_ms": 165.956, "count": 152, "max_ms": 205.368, "p95_ms": 189.09}, "triton_wait": {"avg_ms": 58.405, "count": 153, "max_ms": 85.401, "p95_ms": 81.089}, "visualize": {"avg_ms": 5.207, "count": 153, "max_ms": 6.543, "p95_ms": 6.212}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11821}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10835.559, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 463, "h264_decode_error_count": 1392, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "last_exit_at": "2026-07-01T09:17:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.3, "last_started_at": "2026-07-01T09:17:11+09:00", "pid": 92533, "process": {"exists": false, "pid": 92533}, "restart_count": 508, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 509}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.074, "sample_count": 1674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1418581, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:19:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.087, "last_frame_pushed_at": "2026-07-01T09:19:23+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-01T09:19:23+09:00", "mjpeg": {"age_sec": 0.072, "exists": true, "mtime": "2026-07-01T09:19:23.432735+09:00", "mtime_epoch": 1782865163.4327352, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164316}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:19:23+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5042.086, "name": "celery", "pid": 267, "rss_mb": 5042.086, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 177, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.909, "pushed": 5.909, "read": 5.909}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.799, "count": 176, "max_ms": 41.291, "p95_ms": 35.464}, "gstreamer_push": {"avg_ms": 1.008, "count": 177, "max_ms": 1.538, "p95_ms": 1.079}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.077, "count": 176, "max_ms": 44.581, "p95_ms": 35.986}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.243, "count": 176, "max_ms": 87.48, "p95_ms": 84.497}, "yolo11_640_fire_11class": {"avg_ms": 47.597, "count": 176, "max_ms": 70.014, "p95_ms": 67.727}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.98, "count": 176, "max_ms": 88.187, "p95_ms": 83.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.959, "count": 177, "max_ms": 192.803, "p95_ms": 180.767}, "read_interval": {"avg_ms": 169.664, "count": 177, "max_ms": 202.874, "p95_ms": 191.185}, "triton_wait": {"avg_ms": 63.135, "count": 176, "max_ms": 88.197, "p95_ms": 84.632}, "visualize": {"avg_ms": 5.267, "count": 176, "max_ms": 6.626, "p95_ms": 5.9}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11821}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10835.203, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 463, "h264_decode_error_count": 1392, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "last_exit_at": "2026-07-01T09:19:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.389, "last_started_at": "2026-07-01T09:17:11+09:00", "pid": 92533, "process": {"exists": false, "pid": 92533}, "restart_count": 508, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 509}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.074, "sample_count": 1674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1418582, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:19:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:19:23+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T09:19:23+09:00", "mjpeg": {"age_sec": 0.161, "exists": true, "mtime": "2026-07-01T09:19:23.432735+09:00", "mtime_epoch": 1782865163.4327352, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164316}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:19:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5042.086, "name": "celery", "pid": 267, "rss_mb": 5042.086, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.927, "pushed": 5.927, "read": 5.894}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.817, "count": 177, "max_ms": 41.291, "p95_ms": 35.509}, "gstreamer_push": {"avg_ms": 1.008, "count": 177, "max_ms": 1.538, "p95_ms": 1.083}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.07, "count": 177, "max_ms": 44.581, "p95_ms": 35.986}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.178, "count": 177, "max_ms": 87.48, "p95_ms": 84.497}, "yolo11_640_fire_11class": {"avg_ms": 47.555, "count": 177, "max_ms": 70.014, "p95_ms": 67.727}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.908, "count": 177, "max_ms": 88.187, "p95_ms": 83.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.987, "count": 177, "max_ms": 192.803, "p95_ms": 180.767}, "read_interval": {"avg_ms": 169.68, "count": 176, "max_ms": 202.874, "p95_ms": 191.185}, "triton_wait": {"avg_ms": 63.064, "count": 177, "max_ms": 88.197, "p95_ms": 84.632}, "visualize": {"avg_ms": 5.286, "count": 177, "max_ms": 8.586, "p95_ms": 6.056}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.861, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11821}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10835.32, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 463, "h264_decode_error_count": 1392, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "last_exit_at": "2026-07-01T09:19:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.393, "last_started_at": "2026-07-01T09:17:11+09:00", "pid": 92533, "process": {"exists": false, "pid": 92533}, "restart_count": 508, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 509}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.074, "sample_count": 1674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1418582, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:19:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:19:23+09:00", "last_frame_read_age_sec": 0.171, "last_frame_read_at": "2026-07-01T09:19:23+09:00", "mjpeg": {"age_sec": 0.165, "exists": true, "mtime": "2026-07-01T09:19:23.432735+09:00", "mtime_epoch": 1782865163.4327352, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164316}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:19:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5042.086, "name": "celery", "pid": 267, "rss_mb": 5042.086, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 177, "pushed_frames": 177, "read_frames": 176, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.927, "pushed": 5.927, "read": 5.893}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.817, "count": 177, "max_ms": 41.291, "p95_ms": 35.509}, "gstreamer_push": {"avg_ms": 1.008, "count": 177, "max_ms": 1.538, "p95_ms": 1.083}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.07, "count": 177, "max_ms": 44.581, "p95_ms": 35.986}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.178, "count": 177, "max_ms": 87.48, "p95_ms": 84.497}, "yolo11_640_fire_11class": {"avg_ms": 47.555, "count": 177, "max_ms": 70.014, "p95_ms": 67.727}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.908, "count": 177, "max_ms": 88.187, "p95_ms": 83.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.987, "count": 177, "max_ms": 192.803, "p95_ms": 180.767}, "read_interval": {"avg_ms": 169.68, "count": 176, "max_ms": 202.874, "p95_ms": 191.185}, "triton_wait": {"avg_ms": 63.064, "count": 177, "max_ms": 88.197, "p95_ms": 84.632}, "visualize": {"avg_ms": 5.286, "count": 177, "max_ms": 8.586, "p95_ms": 6.056}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11821}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10943.102, "limit_mb": 8796093022207.996}, "detail": {"pid": 92640, "restart_count": 509, "start_count": 510}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 463, "h264_decode_error_count": 1392, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "last_exit_at": "2026-07-01T09:19:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:19:26+09:00", "pid": 92640, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.113, "name": "ffmpeg", "pid": 92640, "rss_mb": 139.113, "state": "S (sleeping)", "threads": 106}, "restart_count": 509, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 510}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.562, "sample_count": 1650, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1418582, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:19:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.46, "last_frame_pushed_at": "2026-07-01T09:19:23+09:00", "last_frame_read_age_sec": 4.623, "last_frame_read_at": "2026-07-01T09:19:23+09:00", "mjpeg": {"age_sec": 4.445, "exists": true, "mtime": "2026-07-01T09:19:23.604737+09:00", "mtime_epoch": 1782865163.604737, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161427}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:19:28+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5042.086, "name": "celery", "pid": 267, "rss_mb": 5042.086, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 151, "pushed_frames": 151, "read_frames": 150, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.048, "pushed": 5.048, "read": 5.015}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.866, "count": 151, "max_ms": 41.291, "p95_ms": 35.811}, "gstreamer_push": {"avg_ms": 1.008, "count": 151, "max_ms": 1.538, "p95_ms": 1.083}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.884, "count": 151, "max_ms": 41.761, "p95_ms": 34.633}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.163, "count": 151, "max_ms": 87.48, "p95_ms": 84.497}, "yolo11_640_fire_11class": {"avg_ms": 48.557, "count": 151, "max_ms": 70.014, "p95_ms": 67.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.334, "count": 151, "max_ms": 88.187, "p95_ms": 83.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 159.112, "count": 151, "max_ms": 192.803, "p95_ms": 180.742}, "read_interval": {"avg_ms": 169.779, "count": 150, "max_ms": 202.874, "p95_ms": 191.127}, "triton_wait": {"avg_ms": 63.187, "count": 151, "max_ms": 88.197, "p95_ms": 84.632}, "visualize": {"avg_ms": 5.3, "count": 151, "max_ms": 8.586, "p95_ms": 6.119}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11827}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10836.195, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 463, "h264_decode_error_count": 1392, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "last_exit_at": "2026-07-01T09:21:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.091, "last_started_at": "2026-07-01T09:19:26+09:00", "pid": 92640, "process": {"exists": false, "pid": 92640}, "restart_count": 509, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 510}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.571, "sample_count": 1717, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1419339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:21:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T09:21:38+09:00", "last_frame_read_age_sec": 0.161, "last_frame_read_at": "2026-07-01T09:21:37+09:00", "mjpeg": {"age_sec": 0.153, "exists": true, "mtime": "2026-07-01T09:21:37.986279+09:00", "mtime_epoch": 1782865297.9862785, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164235}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:21:38+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5042.344, "name": "celery", "pid": 267, "rss_mb": 5042.344, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 178, "pushed_frames": 178, "read_frames": 177, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.944, "pushed": 5.944, "read": 5.911}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.332, "count": 177, "max_ms": 40.509, "p95_ms": 38.145}, "gstreamer_push": {"avg_ms": 0.993, "count": 178, "max_ms": 1.177, "p95_ms": 1.055}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.688, "count": 177, "max_ms": 42.45, "p95_ms": 36.536}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.996, "count": 177, "max_ms": 83.757, "p95_ms": 80.794}, "yolo11_640_fire_11class": {"avg_ms": 48.532, "count": 177, "max_ms": 65.369, "p95_ms": 63.433}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.507, "count": 177, "max_ms": 84.487, "p95_ms": 80.496}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.582, "count": 178, "max_ms": 182.609, "p95_ms": 176.499}, "read_interval": {"avg_ms": 169.219, "count": 177, "max_ms": 193.156, "p95_ms": 187.218}, "triton_wait": {"avg_ms": 62.519, "count": 177, "max_ms": 84.499, "p95_ms": 81.722}, "visualize": {"avg_ms": 5.251, "count": 177, "max_ms": 8.373, "p95_ms": 5.989}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11827}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10836.156, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 463, "h264_decode_error_count": 1392, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "last_exit_at": "2026-07-01T09:21:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.095, "last_started_at": "2026-07-01T09:19:26+09:00", "pid": 92640, "process": {"exists": false, "pid": 92640}, "restart_count": 509, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 510}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.571, "sample_count": 1717, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1419339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:21:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T09:21:38+09:00", "last_frame_read_age_sec": 0.165, "last_frame_read_at": "2026-07-01T09:21:37+09:00", "mjpeg": {"age_sec": 0.157, "exists": true, "mtime": "2026-07-01T09:21:37.986279+09:00", "mtime_epoch": 1782865297.9862785, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164235}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:21:38+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5042.344, "name": "celery", "pid": 267, "rss_mb": 5042.344, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 178, "pushed_frames": 178, "read_frames": 177, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.943, "pushed": 5.943, "read": 5.91}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.332, "count": 177, "max_ms": 40.509, "p95_ms": 38.145}, "gstreamer_push": {"avg_ms": 0.993, "count": 178, "max_ms": 1.177, "p95_ms": 1.055}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.688, "count": 177, "max_ms": 42.45, "p95_ms": 36.536}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.996, "count": 177, "max_ms": 83.757, "p95_ms": 80.794}, "yolo11_640_fire_11class": {"avg_ms": 48.532, "count": 177, "max_ms": 65.369, "p95_ms": 63.433}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.507, "count": 177, "max_ms": 84.487, "p95_ms": 80.496}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.582, "count": 178, "max_ms": 182.609, "p95_ms": 176.499}, "read_interval": {"avg_ms": 169.219, "count": 177, "max_ms": 193.156, "p95_ms": 187.218}, "triton_wait": {"avg_ms": 62.519, "count": 177, "max_ms": 84.499, "p95_ms": 81.722}, "visualize": {"avg_ms": 5.251, "count": 177, "max_ms": 8.373, "p95_ms": 5.989}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11827}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10944.297, "limit_mb": 8796093022207.996}, "detail": {"pid": 92747, "restart_count": 510, "start_count": 511}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 463, "h264_decode_error_count": 1392, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "last_exit_at": "2026-07-01T09:21:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:21:40+09:00", "pid": 92747, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.188, "name": "ffmpeg", "pid": 92747, "rss_mb": 139.188, "state": "S (sleeping)", "threads": 106}, "restart_count": 510, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 511}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.661, "sample_count": 1689, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1419339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:21:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.783, "last_frame_pushed_at": "2026-07-01T09:21:38+09:00", "last_frame_read_age_sec": 4.939, "last_frame_read_at": "2026-07-01T09:21:37+09:00", "mjpeg": {"age_sec": 4.763, "exists": true, "mtime": "2026-07-01T09:21:38.154280+09:00", "mtime_epoch": 1782865298.1542804, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164203}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:21:42+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5042.344, "name": "celery", "pid": 267, "rss_mb": 5042.344, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.01, "pushed": 5.01, "read": 4.977}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.085, "count": 150, "max_ms": 39.612, "p95_ms": 36.387}, "gstreamer_push": {"avg_ms": 0.994, "count": 150, "max_ms": 1.177, "p95_ms": 1.055}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.729, "count": 150, "max_ms": 42.45, "p95_ms": 36.441}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.912, "count": 150, "max_ms": 83.757, "p95_ms": 81.367}, "yolo11_640_fire_11class": {"avg_ms": 48.636, "count": 150, "max_ms": 65.369, "p95_ms": 63.523}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.76, "count": 150, "max_ms": 84.487, "p95_ms": 81.112}}, "truncated_count": 0}, "processing_total": {"avg_ms": 158.273, "count": 150, "max_ms": 182.609, "p95_ms": 176.403}, "read_interval": {"avg_ms": 168.867, "count": 149, "max_ms": 193.156, "p95_ms": 186.951}, "triton_wait": {"avg_ms": 62.511, "count": 150, "max_ms": 84.499, "p95_ms": 81.741}, "visualize": {"avg_ms": 5.232, "count": 150, "max_ms": 6.957, "p95_ms": 5.989}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11833}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10837.844, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 463, "h264_decode_error_count": 1392, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "last_exit_at": "2026-07-01T09:23:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.617, "last_started_at": "2026-07-01T09:21:40+09:00", "pid": 92747, "process": {"exists": false, "pid": 92747}, "restart_count": 510, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 511}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.064, "sample_count": 1660, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1420047, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:23:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:23:50+09:00", "last_frame_read_age_sec": 0.188, "last_frame_read_at": "2026-07-01T09:23:50+09:00", "mjpeg": {"age_sec": 0.182, "exists": true, "mtime": "2026-07-01T09:23:50.351971+09:00", "mtime_epoch": 1782865430.3519714, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158500}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:23:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5042.859, "name": "celery", "pid": 267, "rss_mb": 5042.859, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 168, "pushed_frames": 168, "read_frames": 167, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.631, "pushed": 5.631, "read": 5.597}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.84, "count": 168, "max_ms": 43.048, "p95_ms": 36.553}, "gstreamer_push": {"avg_ms": 0.926, "count": 168, "max_ms": 1.064, "p95_ms": 1.02}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.669, "count": 168, "max_ms": 36.907, "p95_ms": 33.433}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.007, "count": 168, "max_ms": 86.199, "p95_ms": 82.134}, "yolo11_640_fire_11class": {"avg_ms": 47.488, "count": 168, "max_ms": 64.974, "p95_ms": 62.804}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.76, "count": 168, "max_ms": 82.895, "p95_ms": 80.015}}, "truncated_count": 0}, "processing_total": {"avg_ms": 168.519, "count": 168, "max_ms": 196.688, "p95_ms": 184.545}, "read_interval": {"avg_ms": 178.707, "count": 167, "max_ms": 206.352, "p95_ms": 195.098}, "triton_wait": {"avg_ms": 61.235, "count": 168, "max_ms": 86.211, "p95_ms": 82.145}, "visualize": {"avg_ms": 5.698, "count": 168, "max_ms": 6.966, "p95_ms": 6.444}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.835, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11833}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10837.832, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 463, "h264_decode_error_count": 1392, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "last_exit_at": "2026-07-01T09:23:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.621, "last_started_at": "2026-07-01T09:21:40+09:00", "pid": 92747, "process": {"exists": false, "pid": 92747}, "restart_count": 510, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 511}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.064, "sample_count": 1660, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1420047, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:23:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T09:23:50+09:00", "last_frame_read_age_sec": 0.192, "last_frame_read_at": "2026-07-01T09:23:50+09:00", "mjpeg": {"age_sec": 0.186, "exists": true, "mtime": "2026-07-01T09:23:50.351971+09:00", "mtime_epoch": 1782865430.3519714, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158500}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:23:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5042.859, "name": "celery", "pid": 267, "rss_mb": 5042.859, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 168, "pushed_frames": 168, "read_frames": 167, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.63, "pushed": 5.63, "read": 5.597}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.84, "count": 168, "max_ms": 43.048, "p95_ms": 36.553}, "gstreamer_push": {"avg_ms": 0.926, "count": 168, "max_ms": 1.064, "p95_ms": 1.02}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.669, "count": 168, "max_ms": 36.907, "p95_ms": 33.433}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.007, "count": 168, "max_ms": 86.199, "p95_ms": 82.134}, "yolo11_640_fire_11class": {"avg_ms": 47.488, "count": 168, "max_ms": 64.974, "p95_ms": 62.804}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.76, "count": 168, "max_ms": 82.895, "p95_ms": 80.015}}, "truncated_count": 0}, "processing_total": {"avg_ms": 168.519, "count": 168, "max_ms": 196.688, "p95_ms": 184.545}, "read_interval": {"avg_ms": 178.707, "count": 167, "max_ms": 206.352, "p95_ms": 195.098}, "triton_wait": {"avg_ms": 61.235, "count": 168, "max_ms": 86.211, "p95_ms": 82.145}, "visualize": {"avg_ms": 5.698, "count": 168, "max_ms": 6.966, "p95_ms": 6.444}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.839, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11833}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10943.078, "limit_mb": 8796093022207.996}, "detail": {"pid": 92854, "restart_count": 511, "start_count": 512}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 463, "h264_decode_error_count": 1392, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:19:23+09:00", "last_error_summary": "[h264 @ 0x561470b5adc0] error while decoding MB 10 18, bytestream -11", "last_exit_at": "2026-07-01T09:23:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:23:53+09:00", "pid": 92854, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.441, "name": "ffmpeg", "pid": 92854, "rss_mb": 136.441, "state": "S (sleeping)", "threads": 106}, "restart_count": 511, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 512}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.116, "sample_count": 1633, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1420047, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:23:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.712, "last_frame_pushed_at": "2026-07-01T09:23:50+09:00", "last_frame_read_age_sec": 4.896, "last_frame_read_at": "2026-07-01T09:23:50+09:00", "mjpeg": {"age_sec": 4.699, "exists": true, "mtime": "2026-07-01T09:23:50.543974+09:00", "mtime_epoch": 1782865430.543974, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159453}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:23:55+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5042.859, "name": "celery", "pid": 267, "rss_mb": 5042.859, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.76, "pushed": 4.76, "read": 4.726}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.768, "count": 142, "max_ms": 43.048, "p95_ms": 36.396}, "gstreamer_push": {"avg_ms": 0.932, "count": 142, "max_ms": 1.064, "p95_ms": 1.021}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.711, "count": 142, "max_ms": 36.907, "p95_ms": 33.363}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.237, "count": 142, "max_ms": 83.712, "p95_ms": 79.311}, "yolo11_640_fire_11class": {"avg_ms": 47.487, "count": 142, "max_ms": 64.974, "p95_ms": 58.531}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.851, "count": 142, "max_ms": 82.895, "p95_ms": 79.699}}, "truncated_count": 0}, "processing_total": {"avg_ms": 167.958, "count": 142, "max_ms": 193.462, "p95_ms": 184.24}, "read_interval": {"avg_ms": 178.129, "count": 141, "max_ms": 203.22, "p95_ms": 194.573}, "triton_wait": {"avg_ms": 61.498, "count": 142, "max_ms": 83.725, "p95_ms": 80.21}, "visualize": {"avg_ms": 5.67, "count": 142, "max_ms": 6.966, "p95_ms": 6.437}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.835, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11840}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10839.719, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b4a0439980] error while decoding MB 6 24, bytestream -28", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 464, "h264_decode_error_count": 1393, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:26:04+09:00", "last_error_summary": "[h264 @ 0x55b4a0439980] error while decoding MB 6 24, bytestream -28", "last_exit_at": "2026-07-01T09:23:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.059, "last_started_at": "2026-07-01T09:23:53+09:00", "pid": 92854, "process": {"exists": false, "pid": 92854}, "restart_count": 511, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 512}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 182.514, "sample_count": 1696, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1420835, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:26:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T09:26:04+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T09:26:04+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T09:26:04.257819+09:00", "mtime_epoch": 1782865564.2578194, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165393}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:26:04+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5043.117, "name": "celery", "pid": 267, "rss_mb": 5043.117, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 198, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.567, "pushed": 6.567, "read": 6.6}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.532, "count": 197, "max_ms": 49.611, "p95_ms": 37.663}, "gstreamer_push": {"avg_ms": 1.039, "count": 197, "max_ms": 1.46, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.633, "count": 197, "max_ms": 61.795, "p95_ms": 36.047}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.437, "count": 197, "max_ms": 86.863, "p95_ms": 82.857}, "yolo11_640_fire_11class": {"avg_ms": 48.187, "count": 197, "max_ms": 68.551, "p95_ms": 62.6}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.596, "count": 197, "max_ms": 84.19, "p95_ms": 79.938}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.679, "count": 197, "max_ms": 181.317, "p95_ms": 170.494}, "read_interval": {"avg_ms": 152.055, "count": 198, "max_ms": 193.248, "p95_ms": 182.296}, "triton_wait": {"avg_ms": 62.691, "count": 197, "max_ms": 86.874, "p95_ms": 82.87}, "visualize": {"avg_ms": 4.51, "count": 197, "max_ms": 6.484, "p95_ms": 5.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11840}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10838.664, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:26:04+09:00", "last_error_summary": "[h264 @ 0x55b4a0439980] error while decoding MB 6 24, bytestream -28", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 464, "h264_decode_error_count": 1393, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:26:04+09:00", "last_error_summary": "[h264 @ 0x55b4a0439980] error while decoding MB 6 24, bytestream -28", "last_exit_at": "2026-07-01T09:26:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.175, "last_started_at": "2026-07-01T09:23:53+09:00", "pid": 92854, "process": {"exists": false, "pid": 92854}, "restart_count": 511, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 512}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 182.514, "sample_count": 1697, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1420836, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:26:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:26:04+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T09:26:04+09:00", "mjpeg": {"age_sec": 0.159, "exists": true, "mtime": "2026-07-01T09:26:04.257819+09:00", "mtime_epoch": 1782865564.2578194, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165393}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:26:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5043.117, "name": "celery", "pid": 267, "rss_mb": 5043.117, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 198, "pushed_frames": 198, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.601, "pushed": 6.601, "read": 6.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.531, "count": 197, "max_ms": 49.611, "p95_ms": 37.663}, "gstreamer_push": {"avg_ms": 1.039, "count": 198, "max_ms": 1.46, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.627, "count": 197, "max_ms": 61.795, "p95_ms": 36.047}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.533, "count": 197, "max_ms": 86.863, "p95_ms": 82.857}, "yolo11_640_fire_11class": {"avg_ms": 48.233, "count": 197, "max_ms": 68.551, "p95_ms": 62.6}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.678, "count": 197, "max_ms": 84.19, "p95_ms": 79.938}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.787, "count": 198, "max_ms": 181.317, "p95_ms": 170.494}, "read_interval": {"avg_ms": 152.016, "count": 197, "max_ms": 193.248, "p95_ms": 182.296}, "triton_wait": {"avg_ms": 62.787, "count": 197, "max_ms": 86.874, "p95_ms": 82.87}, "visualize": {"avg_ms": 4.515, "count": 197, "max_ms": 6.484, "p95_ms": 5.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11840}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10838.59, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 464, "h264_decode_error_count": 1393, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:26:04+09:00", "last_error_summary": "[h264 @ 0x55b4a0439980] error while decoding MB 6 24, bytestream -28", "last_exit_at": "2026-07-01T09:26:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.179, "last_started_at": "2026-07-01T09:23:53+09:00", "pid": 92854, "process": {"exists": false, "pid": 92854}, "restart_count": 511, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 512}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 182.514, "sample_count": 1697, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1420836, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:26:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:26:04+09:00", "last_frame_read_age_sec": 0.171, "last_frame_read_at": "2026-07-01T09:26:04+09:00", "mjpeg": {"age_sec": 0.163, "exists": true, "mtime": "2026-07-01T09:26:04.257819+09:00", "mtime_epoch": 1782865564.2578194, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165393}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:26:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5043.117, "name": "celery", "pid": 267, "rss_mb": 5043.117, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 198, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.567, "pushed": 6.6, "read": 6.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.531, "count": 197, "max_ms": 49.611, "p95_ms": 37.663}, "gstreamer_push": {"avg_ms": 1.039, "count": 198, "max_ms": 1.46, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.627, "count": 197, "max_ms": 61.795, "p95_ms": 36.047}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.533, "count": 197, "max_ms": 86.863, "p95_ms": 82.857}, "yolo11_640_fire_11class": {"avg_ms": 48.233, "count": 197, "max_ms": 68.551, "p95_ms": 62.6}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.678, "count": 197, "max_ms": 84.19, "p95_ms": 79.938}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.906, "count": 197, "max_ms": 181.317, "p95_ms": 170.494}, "read_interval": {"avg_ms": 152.016, "count": 197, "max_ms": 193.248, "p95_ms": 182.296}, "triton_wait": {"avg_ms": 62.787, "count": 197, "max_ms": 86.874, "p95_ms": 82.87}, "visualize": {"avg_ms": 4.515, "count": 197, "max_ms": 6.484, "p95_ms": 5.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11840}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10853.988, "limit_mb": 8796093022207.996}, "detail": {"pid": 92961, "restart_count": 512, "start_count": 513}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 464, "h264_decode_error_count": 1393, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:26:04+09:00", "last_error_summary": "[h264 @ 0x55b4a0439980] error while decoding MB 6 24, bytestream -28", "last_exit_at": "2026-07-01T09:26:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:26:06+09:00", "pid": 92961, "process": {"exists": true, "fd_count": 4, "hwm_mb": 52.91, "name": "ffmpeg", "pid": 92961, "rss_mb": 52.91, "state": "S (sleeping)", "threads": 1}, "restart_count": 512, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 513}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 182.558, "sample_count": 1671, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1420836, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:26:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.446, "last_frame_pushed_at": "2026-07-01T09:26:04+09:00", "last_frame_read_age_sec": 4.61, "last_frame_read_at": "2026-07-01T09:26:04+09:00", "mjpeg": {"age_sec": 4.425, "exists": true, "mtime": "2026-07-01T09:26:04.433822+09:00", "mtime_epoch": 1782865564.433822, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161747}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:26:08+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5043.117, "name": "celery", "pid": 267, "rss_mb": 5043.117, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 167, "pushed_frames": 167, "read_frames": 166, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.58, "pushed": 5.58, "read": 5.547}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.592, "count": 167, "max_ms": 49.611, "p95_ms": 37.663}, "gstreamer_push": {"avg_ms": 1.051, "count": 167, "max_ms": 1.46, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.493, "count": 167, "max_ms": 40.358, "p95_ms": 35.376}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.107, "count": 167, "max_ms": 86.863, "p95_ms": 82.897}, "yolo11_640_fire_11class": {"avg_ms": 48.464, "count": 167, "max_ms": 68.551, "p95_ms": 62.6}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.287, "count": 167, "max_ms": 84.19, "p95_ms": 79.938}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.047, "count": 167, "max_ms": 181.317, "p95_ms": 170.494}, "read_interval": {"avg_ms": 153.307, "count": 166, "max_ms": 193.248, "p95_ms": 182.296}, "triton_wait": {"avg_ms": 63.331, "count": 167, "max_ms": 86.874, "p95_ms": 82.909}, "visualize": {"avg_ms": 4.535, "count": 167, "max_ms": 6.484, "p95_ms": 5.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10771.871, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e6ff84dec0] error while decoding MB 117 55, bytestream -9 [in#0/rtsp @ 0x55e6fec0be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6fec0be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 205, "h264_decode_error_count": 211, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:26:55+09:00", "last_error_summary": "[h264 @ 0x55e6ff84dec0] error while decoding MB 117 55, bytestream -9 [in#0/rtsp @ 0x55e6fec0be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6fec0be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T08:57:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 1793.526, "last_started_at": "2026-07-01T08:57:02+09:00", "pid": 91632, "process": {"exists": false, "pid": 91632}, "restart_count": 204, "restart_count_per_hour": 5, "rtsp_timeout_count": 336, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 205}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.645, "sample_count": 2778, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1608426, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:26:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.087, "last_frame_pushed_at": "2026-07-01T09:26:55+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-01T09:26:55+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-01T09:26:55.702559+09:00", "mtime_epoch": 1782865615.7025588, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 214300}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:26:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1819.223, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 23, "pushed_frames": 23, "read_frames": 23, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.769, "pushed": 0.769, "read": 0.769}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.8, "count": 23, "max_ms": 23.459, "p95_ms": 22.954}, "gstreamer_push": {"avg_ms": 1.192, "count": 23, "max_ms": 1.301, "p95_ms": 1.282}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.672, "count": 23, "max_ms": 60.288, "p95_ms": 50.22}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.577, "count": 23, "max_ms": 81.176, "p95_ms": 71.465}, "yolo11_640_fire_11class": {"avg_ms": 49.663, "count": 23, "max_ms": 62.194, "p95_ms": 53.968}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.242, "count": 23, "max_ms": 80.28, "p95_ms": 66.534}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.341, "count": 23, "max_ms": 109.232, "p95_ms": 98.268}, "read_interval": {"avg_ms": 1301.397, "count": 23, "max_ms": 17695.144, "p95_ms": 10159.663}, "triton_wait": {"avg_ms": 55.589, "count": 23, "max_ms": 81.187, "p95_ms": 71.476}, "visualize": {"avg_ms": 3.37, "count": 23, "max_ms": 4.073, "p95_ms": 3.608}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10771.711, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:26:55+09:00", "last_error_summary": "[h264 @ 0x55e6ff84dec0] error while decoding MB 117 55, bytestream -9 [in#0/rtsp @ 0x55e6fec0be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6fec0be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 205, "h264_decode_error_count": 211, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:26:55+09:00", "last_error_summary": "[h264 @ 0x55e6ff84dec0] error while decoding MB 117 55, bytestream -9 [in#0/rtsp @ 0x55e6fec0be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6fec0be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:26:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 1793.549, "last_started_at": "2026-07-01T08:57:02+09:00", "pid": 91632, "process": {"exists": false, "pid": 91632}, "restart_count": 204, "restart_count_per_hour": 5, "rtsp_timeout_count": 336, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 205}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.645, "sample_count": 2778, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1608427, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:26:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T09:26:55+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-01T09:26:55+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-01T09:26:55.702559+09:00", "mtime_epoch": 1782865615.7025588, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 214300}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:26:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1819.223, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 24, "pushed_frames": 24, "read_frames": 23, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.802, "pushed": 0.802, "read": 0.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.901, "count": 24, "max_ms": 23.459, "p95_ms": 23.24}, "gstreamer_push": {"avg_ms": 1.192, "count": 24, "max_ms": 1.301, "p95_ms": 1.282}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.56, "count": 24, "max_ms": 60.288, "p95_ms": 50.22}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.509, "count": 24, "max_ms": 81.176, "p95_ms": 71.465}, "yolo11_640_fire_11class": {"avg_ms": 49.639, "count": 24, "max_ms": 62.194, "p95_ms": 53.968}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.166, "count": 24, "max_ms": 80.28, "p95_ms": 66.534}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.441, "count": 24, "max_ms": 109.232, "p95_ms": 98.268}, "read_interval": {"avg_ms": 1301.397, "count": 23, "max_ms": 17695.144, "p95_ms": 10159.663}, "triton_wait": {"avg_ms": 55.521, "count": 24, "max_ms": 81.187, "p95_ms": 71.476}, "visualize": {"avg_ms": 3.417, "count": 24, "max_ms": 4.491, "p95_ms": 4.073}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10771.688, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 205, "h264_decode_error_count": 211, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:26:55+09:00", "last_error_summary": "[h264 @ 0x55e6ff84dec0] error while decoding MB 117 55, bytestream -9 [in#0/rtsp @ 0x55e6fec0be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6fec0be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:26:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 1793.557, "last_started_at": "2026-07-01T08:57:02+09:00", "pid": 91632, "process": {"exists": false, "pid": 91632}, "restart_count": 204, "restart_count_per_hour": 5, "rtsp_timeout_count": 336, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 205}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.645, "sample_count": 2778, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1608427, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:26:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T09:26:55+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T09:26:55+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-01T09:26:55.702559+09:00", "mtime_epoch": 1782865615.7025588, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 214300}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:26:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1819.223, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 24, "pushed_frames": 24, "read_frames": 23, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.802, "pushed": 0.802, "read": 0.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.901, "count": 24, "max_ms": 23.459, "p95_ms": 23.24}, "gstreamer_push": {"avg_ms": 1.192, "count": 24, "max_ms": 1.301, "p95_ms": 1.282}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.56, "count": 24, "max_ms": 60.288, "p95_ms": 50.22}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.509, "count": 24, "max_ms": 81.176, "p95_ms": 71.465}, "yolo11_640_fire_11class": {"avg_ms": 49.639, "count": 24, "max_ms": 62.194, "p95_ms": 53.968}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.166, "count": 24, "max_ms": 80.28, "p95_ms": 66.534}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.441, "count": 24, "max_ms": 109.232, "p95_ms": 98.268}, "read_interval": {"avg_ms": 1301.397, "count": 23, "max_ms": 17695.144, "p95_ms": 10159.663}, "triton_wait": {"avg_ms": 55.521, "count": 24, "max_ms": 81.187, "p95_ms": 71.476}, "visualize": {"avg_ms": 3.417, "count": 24, "max_ms": 4.491, "p95_ms": 4.073}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10936.945, "limit_mb": 8796093022207.996}, "detail": {"pid": 93068, "restart_count": 205, "start_count": 206}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 205, "h264_decode_error_count": 211, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:26:55+09:00", "last_error_summary": "[h264 @ 0x55e6ff84dec0] error while decoding MB 117 55, bytestream -9 [in#0/rtsp @ 0x55e6fec0be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e6fec0be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:26:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:26:58+09:00", "pid": 93068, "process": {"exists": true, "fd_count": 5, "hwm_mb": 189.969, "name": "ffmpeg", "pid": 93068, "rss_mb": 189.969, "state": "S (sleeping)", "threads": 106}, "restart_count": 205, "restart_count_per_hour": 6, "rtsp_timeout_count": 336, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 206}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.67, "sample_count": 2730, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1608427, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:26:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.648, "last_frame_pushed_at": "2026-07-01T09:26:55+09:00", "last_frame_read_age_sec": 4.737, "last_frame_read_at": "2026-07-01T09:26:55+09:00", "mjpeg": {"age_sec": 4.632, "exists": true, "mtime": "2026-07-01T09:26:55.806560+09:00", "mtime_epoch": 1782865615.8065603, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 216437}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:27:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1819.223, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.872, "pushed": 0.872, "read": 0.872}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.961, "count": 14, "max_ms": 23.459, "p95_ms": 23.459}, "gstreamer_push": {"avg_ms": 1.193, "count": 14, "max_ms": 1.301, "p95_ms": 1.301}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.05, "count": 14, "max_ms": 41.085, "p95_ms": 41.085}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.062, "count": 14, "max_ms": 60.231, "p95_ms": 60.231}, "yolo11_640_fire_11class": {"avg_ms": 48.955, "count": 14, "max_ms": 53.968, "p95_ms": 53.968}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.501, "count": 14, "max_ms": 55.801, "p95_ms": 55.801}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.207, "count": 14, "max_ms": 88.852, "p95_ms": 88.852}, "read_interval": {"avg_ms": 2072.642, "count": 14, "max_ms": 17695.144, "p95_ms": 17695.144}, "triton_wait": {"avg_ms": 54.074, "count": 14, "max_ms": 60.243, "p95_ms": 60.243}, "visualize": {"avg_ms": 3.482, "count": 14, "max_ms": 4.491, "p95_ms": 4.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.059, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11847}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10840.66, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c46c6d1d00] error while decoding MB 4 3, bytestream -8", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 465, "h264_decode_error_count": 1394, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:28:36+09:00", "last_error_summary": "[h264 @ 0x55c46c6d1d00] error while decoding MB 4 3, bytestream -8", "last_exit_at": "2026-07-01T09:26:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.913, "last_started_at": "2026-07-01T09:26:06+09:00", "pid": 92961, "process": {"exists": false, "pid": 92961}, "restart_count": 512, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 513}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.409, "sample_count": 1788, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1421757, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:28:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T09:28:36+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T09:28:36+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T09:28:36.724050+09:00", "mtime_epoch": 1782865716.7240503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163965}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:28:36+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5043.633, "name": "celery", "pid": 267, "rss_mb": 5043.633, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 194, "pushed_frames": 194, "read_frames": 194, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.47, "pushed": 6.47, "read": 6.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.738, "count": 193, "max_ms": 42.26, "p95_ms": 37.842}, "gstreamer_push": {"avg_ms": 1.069, "count": 194, "max_ms": 1.633, "p95_ms": 1.148}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.983, "count": 193, "max_ms": 44.933, "p95_ms": 34.176}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.761, "count": 193, "max_ms": 92.163, "p95_ms": 81.926}, "yolo11_640_fire_11class": {"avg_ms": 49.544, "count": 193, "max_ms": 74.494, "p95_ms": 64.045}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.656, "count": 193, "max_ms": 92.175, "p95_ms": 82.648}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.325, "count": 194, "max_ms": 181.512, "p95_ms": 175.396}, "read_interval": {"avg_ms": 154.902, "count": 194, "max_ms": 192.266, "p95_ms": 185.995}, "triton_wait": {"avg_ms": 63.836, "count": 193, "max_ms": 92.186, "p95_ms": 83.668}, "visualize": {"avg_ms": 4.634, "count": 193, "max_ms": 6.85, "p95_ms": 5.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11847}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10840.465, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:28:36+09:00", "last_error_summary": "[h264 @ 0x55c46c6d1d00] error while decoding MB 4 3, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 465, "h264_decode_error_count": 1394, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:28:36+09:00", "last_error_summary": "[h264 @ 0x55c46c6d1d00] error while decoding MB 4 3, bytestream -8", "last_exit_at": "2026-07-01T09:28:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.028, "last_started_at": "2026-07-01T09:26:06+09:00", "pid": 92961, "process": {"exists": false, "pid": 92961}, "restart_count": 512, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 513}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.409, "sample_count": 1788, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1421758, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:28:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T09:28:36+09:00", "last_frame_read_age_sec": 0.169, "last_frame_read_at": "2026-07-01T09:28:36+09:00", "mjpeg": {"age_sec": 0.162, "exists": true, "mtime": "2026-07-01T09:28:36.724050+09:00", "mtime_epoch": 1782865716.7240503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163965}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:28:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5043.633, "name": "celery", "pid": 267, "rss_mb": 5043.633, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 194, "pushed_frames": 194, "read_frames": 193, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.481, "pushed": 6.481, "read": 6.448}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.746, "count": 194, "max_ms": 42.26, "p95_ms": 37.842}, "gstreamer_push": {"avg_ms": 1.069, "count": 194, "max_ms": 1.633, "p95_ms": 1.148}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.983, "count": 194, "max_ms": 44.933, "p95_ms": 34.176}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.754, "count": 194, "max_ms": 92.163, "p95_ms": 81.926}, "yolo11_640_fire_11class": {"avg_ms": 49.526, "count": 194, "max_ms": 74.494, "p95_ms": 64.045}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.655, "count": 194, "max_ms": 92.175, "p95_ms": 82.648}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.532, "count": 194, "max_ms": 181.512, "p95_ms": 175.396}, "read_interval": {"avg_ms": 155.006, "count": 193, "max_ms": 192.266, "p95_ms": 185.995}, "triton_wait": {"avg_ms": 63.824, "count": 194, "max_ms": 92.186, "p95_ms": 83.668}, "visualize": {"avg_ms": 4.639, "count": 194, "max_ms": 6.85, "p95_ms": 5.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11847}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10840.457, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 465, "h264_decode_error_count": 1394, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:28:36+09:00", "last_error_summary": "[h264 @ 0x55c46c6d1d00] error while decoding MB 4 3, bytestream -8", "last_exit_at": "2026-07-01T09:28:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.032, "last_started_at": "2026-07-01T09:26:06+09:00", "pid": 92961, "process": {"exists": false, "pid": 92961}, "restart_count": 512, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 513}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.409, "sample_count": 1788, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1421758, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:28:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T09:28:36+09:00", "last_frame_read_age_sec": 0.173, "last_frame_read_at": "2026-07-01T09:28:36+09:00", "mjpeg": {"age_sec": 0.166, "exists": true, "mtime": "2026-07-01T09:28:36.724050+09:00", "mtime_epoch": 1782865716.7240503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163965}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:28:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5043.633, "name": "celery", "pid": 267, "rss_mb": 5043.633, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 194, "pushed_frames": 194, "read_frames": 193, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.48, "pushed": 6.48, "read": 6.447}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.746, "count": 194, "max_ms": 42.26, "p95_ms": 37.842}, "gstreamer_push": {"avg_ms": 1.069, "count": 194, "max_ms": 1.633, "p95_ms": 1.148}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.983, "count": 194, "max_ms": 44.933, "p95_ms": 34.176}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.754, "count": 194, "max_ms": 92.163, "p95_ms": 81.926}, "yolo11_640_fire_11class": {"avg_ms": 49.526, "count": 194, "max_ms": 74.494, "p95_ms": 64.045}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.655, "count": 194, "max_ms": 92.175, "p95_ms": 82.648}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.532, "count": 194, "max_ms": 181.512, "p95_ms": 175.396}, "read_interval": {"avg_ms": 155.006, "count": 193, "max_ms": 192.266, "p95_ms": 185.995}, "triton_wait": {"avg_ms": 63.824, "count": 194, "max_ms": 92.186, "p95_ms": 83.668}, "visualize": {"avg_ms": 4.639, "count": 194, "max_ms": 6.85, "p95_ms": 5.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11847}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10946.055, "limit_mb": 8796093022207.996}, "detail": {"pid": 93220, "restart_count": 513, "start_count": 514}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 465, "h264_decode_error_count": 1394, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:28:36+09:00", "last_error_summary": "[h264 @ 0x55c46c6d1d00] error while decoding MB 4 3, bytestream -8", "last_exit_at": "2026-07-01T09:28:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:28:39+09:00", "pid": 93220, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.766, "name": "ffmpeg", "pid": 93220, "rss_mb": 136.766, "state": "S (sleeping)", "threads": 106}, "restart_count": 513, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 514}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.409, "sample_count": 1761, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1421758, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:28:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.681, "last_frame_pushed_at": "2026-07-01T09:28:36+09:00", "last_frame_read_age_sec": 4.844, "last_frame_read_at": "2026-07-01T09:28:36+09:00", "mjpeg": {"age_sec": 4.665, "exists": true, "mtime": "2026-07-01T09:28:36.896053+09:00", "mtime_epoch": 1782865716.8960528, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157417}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:28:41+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5043.633, "name": "celery", "pid": 267, "rss_mb": 5043.633, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 163, "pushed_frames": 163, "read_frames": 162, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.442, "pushed": 5.442, "read": 5.408}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.614, "count": 163, "max_ms": 42.26, "p95_ms": 35.239}, "gstreamer_push": {"avg_ms": 1.065, "count": 163, "max_ms": 1.633, "p95_ms": 1.148}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.903, "count": 163, "max_ms": 44.933, "p95_ms": 34.093}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.072, "count": 163, "max_ms": 85.323, "p95_ms": 81.333}, "yolo11_640_fire_11class": {"avg_ms": 49.173, "count": 163, "max_ms": 66.582, "p95_ms": 63.694}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.472, "count": 163, "max_ms": 86.338, "p95_ms": 82.615}}, "truncated_count": 0}, "processing_total": {"avg_ms": 144.402, "count": 163, "max_ms": 181.512, "p95_ms": 176.446}, "read_interval": {"avg_ms": 155.798, "count": 162, "max_ms": 192.266, "p95_ms": 186.926}, "triton_wait": {"avg_ms": 63.235, "count": 163, "max_ms": 86.351, "p95_ms": 82.66}, "visualize": {"avg_ms": 4.71, "count": 163, "max_ms": 6.85, "p95_ms": 5.514}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10842.383, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562123c1a340] error while decoding MB 34 11, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 466, "h264_decode_error_count": 1395, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:30:51+09:00", "last_error_summary": "[h264 @ 0x562123c1a340] error while decoding MB 34 11, bytestream -7", "last_exit_at": "2026-07-01T09:28:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.114, "last_started_at": "2026-07-01T09:28:39+09:00", "pid": 93220, "process": {"exists": false, "pid": 93220}, "restart_count": 513, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 514}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.823, "sample_count": 1797, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1422549, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:30:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-01T09:30:51+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-01T09:30:51+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-01T09:30:51.622109+09:00", "mtime_epoch": 1782865851.6221094, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161065}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:30:51+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5043.891, "name": "celery", "pid": 267, "rss_mb": 5043.891, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.01, "pushed": 6.01, "read": 6.01}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.421, "count": 179, "max_ms": 48.889, "p95_ms": 36.412}, "gstreamer_push": {"avg_ms": 0.898, "count": 180, "max_ms": 1.908, "p95_ms": 1.025}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.594, "count": 179, "max_ms": 43.034, "p95_ms": 37.568}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.846, "count": 179, "max_ms": 107.071, "p95_ms": 87.398}, "yolo11_640_fire_11class": {"avg_ms": 48.681, "count": 179, "max_ms": 80.917, "p95_ms": 63.557}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.111, "count": 179, "max_ms": 102.373, "p95_ms": 86.598}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.539, "count": 180, "max_ms": 197.489, "p95_ms": 184.662}, "read_interval": {"avg_ms": 166.886, "count": 180, "max_ms": 209.107, "p95_ms": 194.849}, "triton_wait": {"avg_ms": 63.722, "count": 179, "max_ms": 107.083, "p95_ms": 87.702}, "visualize": {"avg_ms": 5.098, "count": 179, "max_ms": 7.364, "p95_ms": 5.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10841.926, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:30:51+09:00", "last_error_summary": "[h264 @ 0x562123c1a340] error while decoding MB 34 11, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 466, "h264_decode_error_count": 1395, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:30:51+09:00", "last_error_summary": "[h264 @ 0x562123c1a340] error while decoding MB 34 11, bytestream -7", "last_exit_at": "2026-07-01T09:30:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.196, "last_started_at": "2026-07-01T09:28:39+09:00", "pid": 93220, "process": {"exists": false, "pid": 93220}, "restart_count": 513, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 514}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.823, "sample_count": 1798, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1422550, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:30:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:30:51+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-01T09:30:51+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-01T09:30:51.622109+09:00", "mtime_epoch": 1782865851.6221094, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161065}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:30:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5043.891, "name": "celery", "pid": 267, "rss_mb": 5043.891, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.032, "pushed": 6.032, "read": 5.998}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.42, "count": 180, "max_ms": 48.889, "p95_ms": 36.136}, "gstreamer_push": {"avg_ms": 0.899, "count": 180, "max_ms": 1.908, "p95_ms": 1.025}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.579, "count": 180, "max_ms": 43.034, "p95_ms": 37.294}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.812, "count": 180, "max_ms": 107.071, "p95_ms": 86.679}, "yolo11_640_fire_11class": {"avg_ms": 48.672, "count": 180, "max_ms": 80.917, "p95_ms": 63.468}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.122, "count": 180, "max_ms": 102.373, "p95_ms": 85.943}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.451, "count": 180, "max_ms": 197.489, "p95_ms": 184.662}, "read_interval": {"avg_ms": 166.92, "count": 179, "max_ms": 209.107, "p95_ms": 195.096}, "triton_wait": {"avg_ms": 63.718, "count": 180, "max_ms": 107.083, "p95_ms": 87.41}, "visualize": {"avg_ms": 5.095, "count": 180, "max_ms": 7.364, "p95_ms": 5.728}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.841, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10841.918, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 466, "h264_decode_error_count": 1395, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:30:51+09:00", "last_error_summary": "[h264 @ 0x562123c1a340] error while decoding MB 34 11, bytestream -7", "last_exit_at": "2026-07-01T09:30:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.201, "last_started_at": "2026-07-01T09:28:39+09:00", "pid": 93220, "process": {"exists": false, "pid": 93220}, "restart_count": 513, "restart_count_per_hour": 22, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 514}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.823, "sample_count": 1798, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1422550, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:30:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T09:30:51+09:00", "last_frame_read_age_sec": 0.146, "last_frame_read_at": "2026-07-01T09:30:51+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-01T09:30:51.622109+09:00", "mtime_epoch": 1782865851.6221094, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161065}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:30:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5043.891, "name": "celery", "pid": 267, "rss_mb": 5043.891, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.031, "pushed": 6.031, "read": 5.997}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.42, "count": 180, "max_ms": 48.889, "p95_ms": 36.136}, "gstreamer_push": {"avg_ms": 0.899, "count": 180, "max_ms": 1.908, "p95_ms": 1.025}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.579, "count": 180, "max_ms": 43.034, "p95_ms": 37.294}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.812, "count": 180, "max_ms": 107.071, "p95_ms": 86.679}, "yolo11_640_fire_11class": {"avg_ms": 48.672, "count": 180, "max_ms": 80.917, "p95_ms": 63.468}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.122, "count": 180, "max_ms": 102.373, "p95_ms": 85.943}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.451, "count": 180, "max_ms": 197.489, "p95_ms": 184.662}, "read_interval": {"avg_ms": 166.92, "count": 179, "max_ms": 209.107, "p95_ms": 195.096}, "triton_wait": {"avg_ms": 63.718, "count": 180, "max_ms": 107.083, "p95_ms": 87.41}, "visualize": {"avg_ms": 5.095, "count": 180, "max_ms": 7.364, "p95_ms": 5.728}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10949.559, "limit_mb": 8796093022207.996}, "detail": {"pid": 93327, "restart_count": 514, "start_count": 515}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 466, "h264_decode_error_count": 1395, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:30:51+09:00", "last_error_summary": "[h264 @ 0x562123c1a340] error while decoding MB 34 11, bytestream -7", "last_exit_at": "2026-07-01T09:30:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:30:53+09:00", "pid": 93327, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.059, "name": "ffmpeg", "pid": 93327, "rss_mb": 139.059, "state": "S (sleeping)", "threads": 106}, "restart_count": 514, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 515}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.823, "sample_count": 1772, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1422550, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:30:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.233, "last_frame_pushed_at": "2026-07-01T09:30:51+09:00", "last_frame_read_age_sec": 4.37, "last_frame_read_at": "2026-07-01T09:30:51+09:00", "mjpeg": {"age_sec": 4.215, "exists": true, "mtime": "2026-07-01T09:30:51.770112+09:00", "mtime_epoch": 1782865851.7701116, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160122}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:30:55+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5043.891, "name": "celery", "pid": 267, "rss_mb": 5043.891, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.179, "pushed": 5.179, "read": 5.146}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.282, "count": 155, "max_ms": 41.531, "p95_ms": 35.374}, "gstreamer_push": {"avg_ms": 0.9, "count": 155, "max_ms": 1.908, "p95_ms": 1.033}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.641, "count": 155, "max_ms": 42.646, "p95_ms": 37.294}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.847, "count": 155, "max_ms": 99.837, "p95_ms": 87.398}, "yolo11_640_fire_11class": {"avg_ms": 48.668, "count": 155, "max_ms": 70.647, "p95_ms": 63.107}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.988, "count": 155, "max_ms": 95.367, "p95_ms": 86.598}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.466, "count": 155, "max_ms": 195.252, "p95_ms": 184.662}, "read_interval": {"avg_ms": 166.929, "count": 154, "max_ms": 205.909, "p95_ms": 194.849}, "triton_wait": {"avg_ms": 63.758, "count": 155, "max_ms": 99.85, "p95_ms": 87.702}, "visualize": {"avg_ms": 5.094, "count": 155, "max_ms": 6.509, "p95_ms": 5.567}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11860}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10844.207, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559d965cbb40] error while decoding MB 26 35, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 467, "h264_decode_error_count": 1396, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:33:06+09:00", "last_error_summary": "[h264 @ 0x559d965cbb40] error while decoding MB 26 35, bytestream -5", "last_exit_at": "2026-07-01T09:30:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.036, "last_started_at": "2026-07-01T09:30:53+09:00", "pid": 93327, "process": {"exists": false, "pid": 93327}, "restart_count": 514, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 515}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.955, "sample_count": 1747, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1423305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:33:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T09:33:05+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T09:33:05+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-01T09:33:05.972221+09:00", "mtime_epoch": 1782865985.9722207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153016}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:33:06+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5044.406, "name": "celery", "pid": 267, "rss_mb": 5044.406, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 181, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.052, "pushed": 6.052, "read": 6.052}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.639, "count": 181, "max_ms": 46.295, "p95_ms": 38.233}, "gstreamer_push": {"avg_ms": 1.058, "count": 181, "max_ms": 1.341, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.882, "count": 181, "max_ms": 49.948, "p95_ms": 39.002}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.965, "count": 181, "max_ms": 95.423, "p95_ms": 92.761}, "yolo11_640_fire_11class": {"avg_ms": 49.758, "count": 181, "max_ms": 71.654, "p95_ms": 65.221}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.217, "count": 181, "max_ms": 93.728, "p95_ms": 88.362}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.163, "count": 181, "max_ms": 193.841, "p95_ms": 184.104}, "read_interval": {"avg_ms": 165.692, "count": 181, "max_ms": 204.814, "p95_ms": 194.561}, "triton_wait": {"avg_ms": 70.526, "count": 181, "max_ms": 95.435, "p95_ms": 93.13}, "visualize": {"avg_ms": 4.75, "count": 181, "max_ms": 6.444, "p95_ms": 5.638}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11860}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10843.469, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:33:06+09:00", "last_error_summary": "[h264 @ 0x559d965cbb40] error while decoding MB 26 35, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 467, "h264_decode_error_count": 1396, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:33:06+09:00", "last_error_summary": "[h264 @ 0x559d965cbb40] error while decoding MB 26 35, bytestream -5", "last_exit_at": "2026-07-01T09:33:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.15, "last_started_at": "2026-07-01T09:30:53+09:00", "pid": 93327, "process": {"exists": false, "pid": 93327}, "restart_count": 514, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 515}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.955, "sample_count": 1747, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1423306, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:33:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:33:06+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T09:33:05+09:00", "mjpeg": {"age_sec": 0.162, "exists": true, "mtime": "2026-07-01T09:33:05.972221+09:00", "mtime_epoch": 1782865985.9722207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153016}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:33:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5044.406, "name": "celery", "pid": 267, "rss_mb": 5044.406, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.058, "pushed": 6.058, "read": 6.025}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.623, "count": 181, "max_ms": 46.295, "p95_ms": 38.233}, "gstreamer_push": {"avg_ms": 1.058, "count": 181, "max_ms": 1.341, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.876, "count": 181, "max_ms": 49.948, "p95_ms": 39.002}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.083, "count": 181, "max_ms": 95.423, "p95_ms": 92.761}, "yolo11_640_fire_11class": {"avg_ms": 49.81, "count": 181, "max_ms": 71.654, "p95_ms": 65.221}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.324, "count": 181, "max_ms": 93.728, "p95_ms": 88.362}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.283, "count": 181, "max_ms": 193.841, "p95_ms": 184.104}, "read_interval": {"avg_ms": 165.775, "count": 180, "max_ms": 204.814, "p95_ms": 194.561}, "triton_wait": {"avg_ms": 70.643, "count": 181, "max_ms": 95.435, "p95_ms": 93.13}, "visualize": {"avg_ms": 4.751, "count": 181, "max_ms": 6.444, "p95_ms": 5.638}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11860}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10843.461, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 467, "h264_decode_error_count": 1396, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:33:06+09:00", "last_error_summary": "[h264 @ 0x559d965cbb40] error while decoding MB 26 35, bytestream -5", "last_exit_at": "2026-07-01T09:33:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.155, "last_started_at": "2026-07-01T09:30:53+09:00", "pid": 93327, "process": {"exists": false, "pid": 93327}, "restart_count": 514, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 515}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.955, "sample_count": 1747, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1423306, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:33:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T09:33:06+09:00", "last_frame_read_age_sec": 0.172, "last_frame_read_at": "2026-07-01T09:33:05+09:00", "mjpeg": {"age_sec": 0.167, "exists": true, "mtime": "2026-07-01T09:33:05.972221+09:00", "mtime_epoch": 1782865985.9722207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153016}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:33:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5044.406, "name": "celery", "pid": 267, "rss_mb": 5044.406, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.057, "pushed": 6.057, "read": 6.024}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.623, "count": 181, "max_ms": 46.295, "p95_ms": 38.233}, "gstreamer_push": {"avg_ms": 1.058, "count": 181, "max_ms": 1.341, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.876, "count": 181, "max_ms": 49.948, "p95_ms": 39.002}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.083, "count": 181, "max_ms": 95.423, "p95_ms": 92.761}, "yolo11_640_fire_11class": {"avg_ms": 49.81, "count": 181, "max_ms": 71.654, "p95_ms": 65.221}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.324, "count": 181, "max_ms": 93.728, "p95_ms": 88.362}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.283, "count": 181, "max_ms": 193.841, "p95_ms": 184.104}, "read_interval": {"avg_ms": 165.775, "count": 180, "max_ms": 204.814, "p95_ms": 194.561}, "triton_wait": {"avg_ms": 70.643, "count": 181, "max_ms": 95.435, "p95_ms": 93.13}, "visualize": {"avg_ms": 4.751, "count": 181, "max_ms": 6.444, "p95_ms": 5.638}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11860}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10959.77, "limit_mb": 8796093022207.996}, "detail": {"pid": 93434, "restart_count": 515, "start_count": 516}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 467, "h264_decode_error_count": 1396, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:33:06+09:00", "last_error_summary": "[h264 @ 0x559d965cbb40] error while decoding MB 26 35, bytestream -5", "last_exit_at": "2026-07-01T09:33:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:33:08+09:00", "pid": 93434, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.578, "name": "ffmpeg", "pid": 93434, "rss_mb": 147.578, "state": "S (sleeping)", "threads": 106}, "restart_count": 515, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 516}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.462, "sample_count": 1719, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1423306, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:33:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.256, "last_frame_pushed_at": "2026-07-01T09:33:06+09:00", "last_frame_read_age_sec": 4.419, "last_frame_read_at": "2026-07-01T09:33:05+09:00", "mjpeg": {"age_sec": 4.242, "exists": true, "mtime": "2026-07-01T09:33:06.144223+09:00", "mtime_epoch": 1782865986.1442232, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153013}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:33:10+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5044.406, "name": "celery", "pid": 267, "rss_mb": 5044.406, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 154, "pushed_frames": 154, "read_frames": 153, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.153, "pushed": 5.153, "read": 5.12}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.789, "count": 154, "max_ms": 46.295, "p95_ms": 39.852}, "gstreamer_push": {"avg_ms": 1.048, "count": 154, "max_ms": 1.219, "p95_ms": 1.146}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.753, "count": 154, "max_ms": 49.948, "p95_ms": 39.018}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.692, "count": 154, "max_ms": 95.423, "p95_ms": 92.471}, "yolo11_640_fire_11class": {"avg_ms": 49.821, "count": 154, "max_ms": 71.654, "p95_ms": 65.358}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.216, "count": 154, "max_ms": 93.728, "p95_ms": 88.072}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.122, "count": 154, "max_ms": 193.841, "p95_ms": 184.459}, "read_interval": {"avg_ms": 167.505, "count": 153, "max_ms": 204.814, "p95_ms": 195.736}, "triton_wait": {"avg_ms": 70.33, "count": 154, "max_ms": 95.435, "p95_ms": 93.13}, "visualize": {"avg_ms": 4.832, "count": 154, "max_ms": 6.444, "p95_ms": 5.723}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11867}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10845.461, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d7c6633180] left block unavailable for requested intra mode [h264 @ 0x55d7c6633180] error while decoding MB 0 17, bytestream 9587 [h264 @ 0x55d7c5aae780] error while decoding MB 4 37, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 468, "h264_decode_error_count": 1398, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:35:26+09:00", "last_error_summary": "[h264 @ 0x55d7c6633180] left block unavailable for requested intra mode [h264 @ 0x55d7c6633180] error while decoding MB 0 17, bytestream 9587 [h264 @ 0x55d7c5aae780] error while decoding MB 4 37, bytestream -5", "last_exit_at": "2026-07-01T09:33:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.827, "last_started_at": "2026-07-01T09:33:08+09:00", "pid": 93434, "process": {"exists": false, "pid": 93434}, "restart_count": 515, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 516}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 194.016, "sample_count": 1701, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1424098, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:35:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T09:35:26+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T09:35:26+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T09:35:26.166473+09:00", "mtime_epoch": 1782866126.166473, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155028}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:35:26+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5044.922, "name": "celery", "pid": 267, "rss_mb": 5044.922, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 166, "pushed_frames": 166, "read_frames": 166, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.559, "pushed": 5.559, "read": 5.559}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.568, "count": 166, "max_ms": 51.935, "p95_ms": 42.032}, "gstreamer_push": {"avg_ms": 1.051, "count": 166, "max_ms": 1.522, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.999, "count": 166, "max_ms": 48.88, "p95_ms": 40.532}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.093, "count": 166, "max_ms": 221.416, "p95_ms": 146.535}, "yolo11_640_fire_11class": {"avg_ms": 53.355, "count": 166, "max_ms": 91.614, "p95_ms": 72.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.306, "count": 166, "max_ms": 170.49, "p95_ms": 118.073}}, "truncated_count": 0}, "processing_total": {"avg_ms": 169.305, "count": 166, "max_ms": 292.951, "p95_ms": 232.04}, "read_interval": {"avg_ms": 180.709, "count": 166, "max_ms": 306.436, "p95_ms": 243.697}, "triton_wait": {"avg_ms": 85.313, "count": 166, "max_ms": 221.427, "p95_ms": 146.547}, "visualize": {"avg_ms": 4.747, "count": 166, "max_ms": 6.82, "p95_ms": 5.703}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.862, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11867}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10844.848, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:35:26+09:00", "last_error_summary": "[h264 @ 0x55d7c6633180] left block unavailable for requested intra mode [h264 @ 0x55d7c6633180] error while decoding MB 0 17, bytestream 9587 [h264 @ 0x55d7c5aae780] error while decoding MB 4 37, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 468, "h264_decode_error_count": 1398, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:35:26+09:00", "last_error_summary": "[h264 @ 0x55d7c6633180] left block unavailable for requested intra mode [h264 @ 0x55d7c6633180] error while decoding MB 0 17, bytestream 9587 [h264 @ 0x55d7c5aae780] error while decoding MB 4 37, bytestream -5", "last_exit_at": "2026-07-01T09:35:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.946, "last_started_at": "2026-07-01T09:33:08+09:00", "pid": 93434, "process": {"exists": false, "pid": 93434}, "restart_count": 515, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 516}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 194.016, "sample_count": 1702, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1424099, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:35:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:35:26+09:00", "last_frame_read_age_sec": 0.171, "last_frame_read_at": "2026-07-01T09:35:26+09:00", "mjpeg": {"age_sec": 0.164, "exists": true, "mtime": "2026-07-01T09:35:26.166473+09:00", "mtime_epoch": 1782866126.166473, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155028}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:35:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5044.922, "name": "celery", "pid": 267, "rss_mb": 5044.922, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 167, "pushed_frames": 167, "read_frames": 166, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.57, "pushed": 5.57, "read": 5.537}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.585, "count": 166, "max_ms": 51.935, "p95_ms": 42.032}, "gstreamer_push": {"avg_ms": 1.051, "count": 167, "max_ms": 1.522, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.99, "count": 166, "max_ms": 48.88, "p95_ms": 40.532}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.913, "count": 166, "max_ms": 221.416, "p95_ms": 146.535}, "yolo11_640_fire_11class": {"avg_ms": 53.206, "count": 166, "max_ms": 91.614, "p95_ms": 72.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.169, "count": 166, "max_ms": 170.49, "p95_ms": 118.073}}, "truncated_count": 0}, "processing_total": {"avg_ms": 169.283, "count": 167, "max_ms": 292.951, "p95_ms": 232.04}, "read_interval": {"avg_ms": 180.709, "count": 166, "max_ms": 306.436, "p95_ms": 243.697}, "triton_wait": {"avg_ms": 85.154, "count": 166, "max_ms": 221.427, "p95_ms": 146.547}, "visualize": {"avg_ms": 4.748, "count": 166, "max_ms": 6.82, "p95_ms": 5.703}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11867}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10844.84, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 468, "h264_decode_error_count": 1398, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:35:26+09:00", "last_error_summary": "[h264 @ 0x55d7c6633180] left block unavailable for requested intra mode [h264 @ 0x55d7c6633180] error while decoding MB 0 17, bytestream 9587 [h264 @ 0x55d7c5aae780] error while decoding MB 4 37, bytestream -5", "last_exit_at": "2026-07-01T09:35:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.95, "last_started_at": "2026-07-01T09:33:08+09:00", "pid": 93434, "process": {"exists": false, "pid": 93434}, "restart_count": 515, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 516}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 194.016, "sample_count": 1702, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1424099, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:35:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:35:26+09:00", "last_frame_read_age_sec": 0.175, "last_frame_read_at": "2026-07-01T09:35:26+09:00", "mjpeg": {"age_sec": 0.168, "exists": true, "mtime": "2026-07-01T09:35:26.166473+09:00", "mtime_epoch": 1782866126.166473, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155028}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:35:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5044.922, "name": "celery", "pid": 267, "rss_mb": 5044.922, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 167, "pushed_frames": 167, "read_frames": 166, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.57, "pushed": 5.57, "read": 5.536}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.585, "count": 166, "max_ms": 51.935, "p95_ms": 42.032}, "gstreamer_push": {"avg_ms": 1.051, "count": 167, "max_ms": 1.522, "p95_ms": 1.174}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.99, "count": 166, "max_ms": 48.88, "p95_ms": 40.532}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.913, "count": 166, "max_ms": 221.416, "p95_ms": 146.535}, "yolo11_640_fire_11class": {"avg_ms": 53.206, "count": 166, "max_ms": 91.614, "p95_ms": 72.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.169, "count": 166, "max_ms": 170.49, "p95_ms": 118.073}}, "truncated_count": 0}, "processing_total": {"avg_ms": 169.283, "count": 167, "max_ms": 292.951, "p95_ms": 232.04}, "read_interval": {"avg_ms": 180.709, "count": 166, "max_ms": 306.436, "p95_ms": 243.697}, "triton_wait": {"avg_ms": 85.154, "count": 166, "max_ms": 221.427, "p95_ms": 146.547}, "visualize": {"avg_ms": 4.748, "count": 166, "max_ms": 6.82, "p95_ms": 5.703}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11867}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10950.473, "limit_mb": 8796093022207.996}, "detail": {"pid": 93541, "restart_count": 516, "start_count": 517}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 468, "h264_decode_error_count": 1398, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:35:26+09:00", "last_error_summary": "[h264 @ 0x55d7c6633180] left block unavailable for requested intra mode [h264 @ 0x55d7c6633180] error while decoding MB 0 17, bytestream 9587 [h264 @ 0x55d7c5aae780] error while decoding MB 4 37, bytestream -5", "last_exit_at": "2026-07-01T09:35:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:35:28+09:00", "pid": 93541, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.527, "name": "ffmpeg", "pid": 93541, "rss_mb": 136.527, "state": "S (sleeping)", "threads": 106}, "restart_count": 516, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 517}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 194.134, "sample_count": 1673, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1424099, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:35:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.634, "last_frame_pushed_at": "2026-07-01T09:35:26+09:00", "last_frame_read_age_sec": 4.802, "last_frame_read_at": "2026-07-01T09:35:26+09:00", "mjpeg": {"age_sec": 4.619, "exists": true, "mtime": "2026-07-01T09:35:26.342476+09:00", "mtime_epoch": 1782866126.342476, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155045}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:35:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5044.922, "name": "celery", "pid": 267, "rss_mb": 5044.922, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 145, "pushed_frames": 145, "read_frames": 144, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.834, "pushed": 4.834, "read": 4.801}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.823, "count": 144, "max_ms": 51.935, "p95_ms": 43.534}, "gstreamer_push": {"avg_ms": 1.058, "count": 145, "max_ms": 1.522, "p95_ms": 1.18}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.548, "count": 144, "max_ms": 44.996, "p95_ms": 39.92}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.119, "count": 144, "max_ms": 193.758, "p95_ms": 115.647}, "yolo11_640_fire_11class": {"avg_ms": 52.919, "count": 144, "max_ms": 91.614, "p95_ms": 71.7}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.781, "count": 144, "max_ms": 170.49, "p95_ms": 106.586}}, "truncated_count": 0}, "processing_total": {"avg_ms": 164.58, "count": 145, "max_ms": 263.323, "p95_ms": 196.733}, "read_interval": {"avg_ms": 176.029, "count": 144, "max_ms": 276.312, "p95_ms": 207.161}, "triton_wait": {"avg_ms": 81.376, "count": 144, "max_ms": 193.77, "p95_ms": 115.659}, "visualize": {"avg_ms": 4.708, "count": 144, "max_ms": 6.257, "p95_ms": 5.556}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13444}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10776.953, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5630b49d1740] error while decoding MB 17 35, bytestream -8 [in#0/rtsp @ 0x5630b3bece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5630b3bece00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 206, "h264_decode_error_count": 212, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:36:02+09:00", "last_error_summary": "[h264 @ 0x5630b49d1740] error while decoding MB 17 35, bytestream -8 [in#0/rtsp @ 0x5630b3bece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5630b3bece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:26:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 543.567, "last_started_at": "2026-07-01T09:26:58+09:00", "pid": 93068, "process": {"exists": false, "pid": 93068}, "restart_count": 205, "restart_count_per_hour": 4, "rtsp_timeout_count": 338, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 206}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.537, "sample_count": 2541, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1613355, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:36:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-01T09:36:01+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-01T09:36:01+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-01T09:36:01.947054+09:00", "mtime_epoch": 1782866161.9470544, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 226937}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:36:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1821.543, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 50, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.668, "pushed": 1.668, "read": 1.668}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.061, "count": 50, "max_ms": 28.845, "p95_ms": 24.772}, "gstreamer_push": {"avg_ms": 1.201, "count": 50, "max_ms": 1.393, "p95_ms": 1.307}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.657, "count": 50, "max_ms": 62.845, "p95_ms": 55.34}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.609, "count": 50, "max_ms": 103.432, "p95_ms": 93.964}, "yolo11_640_fire_11class": {"avg_ms": 55.551, "count": 50, "max_ms": 72.06, "p95_ms": 65.993}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.846, "count": 50, "max_ms": 95.293, "p95_ms": 85.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.803, "count": 50, "max_ms": 132.299, "p95_ms": 122.901}, "read_interval": {"avg_ms": 599.969, "count": 50, "max_ms": 15733.707, "p95_ms": 145.203}, "triton_wait": {"avg_ms": 64.788, "count": 50, "max_ms": 103.447, "p95_ms": 93.974}, "visualize": {"avg_ms": 3.366, "count": 50, "max_ms": 3.641, "p95_ms": 3.621}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13444}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10777.004, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:36:02+09:00", "last_error_summary": "[h264 @ 0x5630b49d1740] error while decoding MB 17 35, bytestream -8 [in#0/rtsp @ 0x5630b3bece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5630b3bece00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 206, "h264_decode_error_count": 212, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:36:02+09:00", "last_error_summary": "[h264 @ 0x5630b49d1740] error while decoding MB 17 35, bytestream -8 [in#0/rtsp @ 0x5630b3bece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5630b3bece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:36:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 543.612, "last_started_at": "2026-07-01T09:26:58+09:00", "pid": 93068, "process": {"exists": false, "pid": 93068}, "restart_count": 205, "restart_count_per_hour": 4, "rtsp_timeout_count": 338, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 206}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.537, "sample_count": 2541, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1613356, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:36:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T09:36:02+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-01T09:36:01+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-01T09:36:01.947054+09:00", "mtime_epoch": 1782866161.9470544, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 226937}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:36:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1821.543, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.671, "pushed": 1.671, "read": 1.638}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.132, "count": 50, "max_ms": 28.845, "p95_ms": 24.772}, "gstreamer_push": {"avg_ms": 1.205, "count": 50, "max_ms": 1.393, "p95_ms": 1.342}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.635, "count": 50, "max_ms": 62.845, "p95_ms": 55.34}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.783, "count": 50, "max_ms": 103.432, "p95_ms": 93.964}, "yolo11_640_fire_11class": {"avg_ms": 55.617, "count": 50, "max_ms": 72.06, "p95_ms": 65.993}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.908, "count": 50, "max_ms": 95.293, "p95_ms": 85.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.076, "count": 50, "max_ms": 132.299, "p95_ms": 122.901}, "read_interval": {"avg_ms": 610.132, "count": 49, "max_ms": 15733.707, "p95_ms": 145.203}, "triton_wait": {"avg_ms": 64.962, "count": 50, "max_ms": 103.447, "p95_ms": 93.974}, "visualize": {"avg_ms": 3.391, "count": 50, "max_ms": 4.566, "p95_ms": 3.633}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13444}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10776.996, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 206, "h264_decode_error_count": 212, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:36:02+09:00", "last_error_summary": "[h264 @ 0x5630b49d1740] error while decoding MB 17 35, bytestream -8 [in#0/rtsp @ 0x5630b3bece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5630b3bece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:36:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 543.62, "last_started_at": "2026-07-01T09:26:58+09:00", "pid": 93068, "process": {"exists": false, "pid": 93068}, "restart_count": 205, "restart_count_per_hour": 4, "rtsp_timeout_count": 338, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 206}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.537, "sample_count": 2541, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1613356, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:36:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T09:36:02+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-01T09:36:01+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-01T09:36:01.947054+09:00", "mtime_epoch": 1782866161.9470544, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 226937}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:36:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1821.543, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.671, "pushed": 1.671, "read": 1.637}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.132, "count": 50, "max_ms": 28.845, "p95_ms": 24.772}, "gstreamer_push": {"avg_ms": 1.205, "count": 50, "max_ms": 1.393, "p95_ms": 1.342}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.635, "count": 50, "max_ms": 62.845, "p95_ms": 55.34}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.783, "count": 50, "max_ms": 103.432, "p95_ms": 93.964}, "yolo11_640_fire_11class": {"avg_ms": 55.617, "count": 50, "max_ms": 72.06, "p95_ms": 65.993}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.908, "count": 50, "max_ms": 95.293, "p95_ms": 85.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.076, "count": 50, "max_ms": 132.299, "p95_ms": 122.901}, "read_interval": {"avg_ms": 610.132, "count": 49, "max_ms": 15733.707, "p95_ms": 145.203}, "triton_wait": {"avg_ms": 64.962, "count": 50, "max_ms": 103.447, "p95_ms": 93.974}, "visualize": {"avg_ms": 3.391, "count": 50, "max_ms": 4.566, "p95_ms": 3.633}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 68, "tracking_summary": 13444}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10892.734, "limit_mb": 8796093022207.996}, "detail": {"pid": 93648, "restart_count": 206, "start_count": 207}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 206, "h264_decode_error_count": 212, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:36:02+09:00", "last_error_summary": "[h264 @ 0x5630b49d1740] error while decoding MB 17 35, bytestream -8 [in#0/rtsp @ 0x5630b3bece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5630b3bece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:36:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:36:04+09:00", "pid": 93648, "process": {"exists": true, "fd_count": 5, "hwm_mb": 148.324, "name": "ffmpeg", "pid": 93648, "rss_mb": 148.324, "state": "R (running)", "threads": 49}, "restart_count": 206, "restart_count_per_hour": 5, "rtsp_timeout_count": 338, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 207}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 122.742, "sample_count": 2498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1613356, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:36:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.231, "last_frame_pushed_at": "2026-07-01T09:36:02+09:00", "last_frame_read_age_sec": 4.334, "last_frame_read_at": "2026-07-01T09:36:01+09:00", "mjpeg": {"age_sec": 4.212, "exists": true, "mtime": "2026-07-01T09:36:02.063056+09:00", "mtime_epoch": 1782866162.0630562, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 225996}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:36:06+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1821.543, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.611, "pushed": 0.611, "read": 0.611}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.268, "count": 13, "max_ms": 24.346, "p95_ms": 24.346}, "gstreamer_push": {"avg_ms": 1.213, "count": 13, "max_ms": 1.358, "p95_ms": 1.358}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.004, "count": 13, "max_ms": 62.845, "p95_ms": 62.845}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.034, "count": 13, "max_ms": 103.432, "p95_ms": 103.432}, "yolo11_640_fire_11class": {"avg_ms": 55.712, "count": 13, "max_ms": 72.06, "p95_ms": 72.06}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.194, "count": 13, "max_ms": 82.488, "p95_ms": 82.488}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.364, "count": 13, "max_ms": 132.299, "p95_ms": 132.299}, "read_interval": {"avg_ms": 1998.204, "count": 13, "max_ms": 15733.707, "p95_ms": 15733.707}, "triton_wait": {"avg_ms": 66.047, "count": 13, "max_ms": 103.447, "p95_ms": 103.447}, "visualize": {"avg_ms": 3.467, "count": 13, "max_ms": 4.566, "p95_ms": 4.566}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.285, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11875}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10853.184, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562ca5212bc0] error while decoding MB 4 38, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 469, "h264_decode_error_count": 1399, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:37:55+09:00", "last_error_summary": "[h264 @ 0x562ca5212bc0] error while decoding MB 4 38, bytestream -9", "last_exit_at": "2026-07-01T09:35:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.76, "last_started_at": "2026-07-01T09:35:28+09:00", "pid": 93541, "process": {"exists": false, "pid": 93541}, "restart_count": 516, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 517}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.818, "sample_count": 1766, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1425009, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:37:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.055, "last_frame_pushed_at": "2026-07-01T09:37:55+09:00", "last_frame_read_age_sec": 0.047, "last_frame_read_at": "2026-07-01T09:37:55+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T09:37:55.680917+09:00", "mtime_epoch": 1782866275.6809173, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156164}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:37:55+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5045.18, "name": "celery", "pid": 267, "rss_mb": 5045.18, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 192, "pushed_frames": 192, "read_frames": 192, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.41, "pushed": 6.41, "read": 6.41}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.812, "count": 191, "max_ms": 36.806, "p95_ms": 34.221}, "gstreamer_push": {"avg_ms": 0.818, "count": 192, "max_ms": 1.206, "p95_ms": 1.119}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.047, "count": 191, "max_ms": 42.87, "p95_ms": 35.245}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.704, "count": 191, "max_ms": 95.713, "p95_ms": 83.326}, "yolo11_640_fire_11class": {"avg_ms": 48.547, "count": 191, "max_ms": 71.233, "p95_ms": 64.455}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.546, "count": 191, "max_ms": 91.341, "p95_ms": 83.43}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.398, "count": 192, "max_ms": 188.748, "p95_ms": 173.898}, "read_interval": {"avg_ms": 156.606, "count": 192, "max_ms": 198.086, "p95_ms": 182.743}, "triton_wait": {"avg_ms": 61.508, "count": 191, "max_ms": 95.726, "p95_ms": 83.954}, "visualize": {"avg_ms": 4.871, "count": 191, "max_ms": 6.186, "p95_ms": 5.819}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11875}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10852.0, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:37:55+09:00", "last_error_summary": "[h264 @ 0x562ca5212bc0] error while decoding MB 4 38, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 469, "h264_decode_error_count": 1399, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:37:55+09:00", "last_error_summary": "[h264 @ 0x562ca5212bc0] error while decoding MB 4 38, bytestream -9", "last_exit_at": "2026-07-01T09:37:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.862, "last_started_at": "2026-07-01T09:35:28+09:00", "pid": 93541, "process": {"exists": false, "pid": 93541}, "restart_count": 516, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 517}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.818, "sample_count": 1766, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1425010, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:37:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T09:37:55+09:00", "last_frame_read_age_sec": 0.15, "last_frame_read_at": "2026-07-01T09:37:55+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-01T09:37:55.680917+09:00", "mtime_epoch": 1782866275.6809173, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156164}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:37:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5045.18, "name": "celery", "pid": 267, "rss_mb": 5045.18, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 192, "pushed_frames": 192, "read_frames": 191, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.426, "pushed": 6.426, "read": 6.392}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.816, "count": 192, "max_ms": 36.806, "p95_ms": 34.221}, "gstreamer_push": {"avg_ms": 0.817, "count": 192, "max_ms": 1.206, "p95_ms": 1.119}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.035, "count": 192, "max_ms": 42.87, "p95_ms": 35.245}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.68, "count": 192, "max_ms": 95.713, "p95_ms": 83.326}, "yolo11_640_fire_11class": {"avg_ms": 48.528, "count": 192, "max_ms": 71.233, "p95_ms": 64.455}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.52, "count": 192, "max_ms": 91.341, "p95_ms": 83.43}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.297, "count": 192, "max_ms": 188.748, "p95_ms": 173.898}, "read_interval": {"avg_ms": 156.516, "count": 191, "max_ms": 198.086, "p95_ms": 182.743}, "triton_wait": {"avg_ms": 61.481, "count": 192, "max_ms": 95.726, "p95_ms": 83.954}, "visualize": {"avg_ms": 4.872, "count": 192, "max_ms": 6.186, "p95_ms": 5.819}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11875}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10851.816, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 469, "h264_decode_error_count": 1399, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:37:55+09:00", "last_error_summary": "[h264 @ 0x562ca5212bc0] error while decoding MB 4 38, bytestream -9", "last_exit_at": "2026-07-01T09:37:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.867, "last_started_at": "2026-07-01T09:35:28+09:00", "pid": 93541, "process": {"exists": false, "pid": 93541}, "restart_count": 516, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 517}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.818, "sample_count": 1766, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1425010, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:37:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:37:55+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-01T09:37:55+09:00", "mjpeg": {"age_sec": 0.146, "exists": true, "mtime": "2026-07-01T09:37:55.680917+09:00", "mtime_epoch": 1782866275.6809173, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156164}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:37:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5045.18, "name": "celery", "pid": 267, "rss_mb": 5045.18, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 192, "pushed_frames": 192, "read_frames": 191, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.425, "pushed": 6.425, "read": 6.391}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.816, "count": 192, "max_ms": 36.806, "p95_ms": 34.221}, "gstreamer_push": {"avg_ms": 0.817, "count": 192, "max_ms": 1.206, "p95_ms": 1.119}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.035, "count": 192, "max_ms": 42.87, "p95_ms": 35.245}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.68, "count": 192, "max_ms": 95.713, "p95_ms": 83.326}, "yolo11_640_fire_11class": {"avg_ms": 48.528, "count": 192, "max_ms": 71.233, "p95_ms": 64.455}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.52, "count": 192, "max_ms": 91.341, "p95_ms": 83.43}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.297, "count": 192, "max_ms": 188.748, "p95_ms": 173.898}, "read_interval": {"avg_ms": 156.516, "count": 191, "max_ms": 198.086, "p95_ms": 182.743}, "triton_wait": {"avg_ms": 61.481, "count": 192, "max_ms": 95.726, "p95_ms": 83.954}, "visualize": {"avg_ms": 4.872, "count": 192, "max_ms": 6.186, "p95_ms": 5.819}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11875}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10963.449, "limit_mb": 8796093022207.996}, "detail": {"pid": 93860, "restart_count": 517, "start_count": 518}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 469, "h264_decode_error_count": 1399, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:37:55+09:00", "last_error_summary": "[h264 @ 0x562ca5212bc0] error while decoding MB 4 38, bytestream -9", "last_exit_at": "2026-07-01T09:37:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:37:57+09:00", "pid": 93860, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.742, "name": "ffmpeg", "pid": 93860, "rss_mb": 141.742, "state": "S (sleeping)", "threads": 106}, "restart_count": 517, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 518}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.818, "sample_count": 1742, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1425010, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:37:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.141, "last_frame_pushed_at": "2026-07-01T09:37:55+09:00", "last_frame_read_age_sec": 4.287, "last_frame_read_at": "2026-07-01T09:37:55+09:00", "mjpeg": {"age_sec": 4.124, "exists": true, "mtime": "2026-07-01T09:37:55.836920+09:00", "mtime_epoch": 1782866275.8369198, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155914}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:37:59+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5045.18, "name": "celery", "pid": 267, "rss_mb": 5045.18, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 167, "pushed_frames": 167, "read_frames": 166, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.585, "pushed": 5.585, "read": 5.551}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.495, "count": 167, "max_ms": 36.806, "p95_ms": 33.071}, "gstreamer_push": {"avg_ms": 0.783, "count": 167, "max_ms": 1.206, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.842, "count": 167, "max_ms": 42.87, "p95_ms": 35.245}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.2, "count": 167, "max_ms": 88.398, "p95_ms": 77.064}, "yolo11_640_fire_11class": {"avg_ms": 47.742, "count": 167, "max_ms": 69.87, "p95_ms": 61.779}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.27, "count": 167, "max_ms": 88.41, "p95_ms": 79.278}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.34, "count": 167, "max_ms": 188.748, "p95_ms": 170.452}, "read_interval": {"avg_ms": 155.376, "count": 166, "max_ms": 198.086, "p95_ms": 181.051}, "triton_wait": {"avg_ms": 60.012, "count": 167, "max_ms": 88.42, "p95_ms": 79.831}, "visualize": {"avg_ms": 4.893, "count": 167, "max_ms": 6.186, "p95_ms": 5.942}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11881}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10854.648, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5572ddca8580] error while decoding MB 8 7, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 470, "h264_decode_error_count": 1400, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:40:09+09:00", "last_error_summary": "[h264 @ 0x5572ddca8580] error while decoding MB 8 7, bytestream -7", "last_exit_at": "2026-07-01T09:37:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.834, "last_started_at": "2026-07-01T09:37:57+09:00", "pid": 93860, "process": {"exists": false, "pid": 93860}, "restart_count": 517, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 518}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.337, "sample_count": 1743, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1425749, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:40:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T09:40:09+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T09:40:09+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-01T09:40:09.755137+09:00", "mtime_epoch": 1782866409.755137, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157423}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:40:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5045.438, "name": "celery", "pid": 267, "rss_mb": 5045.438, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 175, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.845, "pushed": 5.845, "read": 5.845}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.479, "count": 175, "max_ms": 38.685, "p95_ms": 36.704}, "gstreamer_push": {"avg_ms": 1.005, "count": 175, "max_ms": 1.457, "p95_ms": 1.066}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.746, "count": 175, "max_ms": 43.769, "p95_ms": 39.004}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.297, "count": 175, "max_ms": 93.45, "p95_ms": 89.835}, "yolo11_640_fire_11class": {"avg_ms": 50.612, "count": 175, "max_ms": 82.039, "p95_ms": 69.431}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.166, "count": 175, "max_ms": 97.199, "p95_ms": 88.3}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.936, "count": 175, "max_ms": 195.496, "p95_ms": 183.875}, "read_interval": {"avg_ms": 171.687, "count": 175, "max_ms": 205.956, "p95_ms": 194.265}, "triton_wait": {"avg_ms": 65.403, "count": 175, "max_ms": 97.211, "p95_ms": 89.992}, "visualize": {"avg_ms": 5.177, "count": 175, "max_ms": 5.978, "p95_ms": 5.592}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11881}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10853.82, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:40:09+09:00", "last_error_summary": "[h264 @ 0x5572ddca8580] error while decoding MB 8 7, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 470, "h264_decode_error_count": 1400, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:40:09+09:00", "last_error_summary": "[h264 @ 0x5572ddca8580] error while decoding MB 8 7, bytestream -7", "last_exit_at": "2026-07-01T09:40:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.947, "last_started_at": "2026-07-01T09:37:57+09:00", "pid": 93860, "process": {"exists": false, "pid": 93860}, "restart_count": 517, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 518}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.337, "sample_count": 1744, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1425750, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:40:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:40:09+09:00", "last_frame_read_age_sec": 0.162, "last_frame_read_at": "2026-07-01T09:40:09+09:00", "mjpeg": {"age_sec": 0.152, "exists": true, "mtime": "2026-07-01T09:40:09.755137+09:00", "mtime_epoch": 1782866409.755137, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157423}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:40:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5045.438, "name": "celery", "pid": 267, "rss_mb": 5045.438, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.859, "pushed": 5.859, "read": 5.825}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.466, "count": 175, "max_ms": 38.685, "p95_ms": 36.704}, "gstreamer_push": {"avg_ms": 1.006, "count": 175, "max_ms": 1.457, "p95_ms": 1.066}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.74, "count": 175, "max_ms": 43.769, "p95_ms": 39.004}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.299, "count": 175, "max_ms": 93.45, "p95_ms": 89.835}, "yolo11_640_fire_11class": {"avg_ms": 50.585, "count": 175, "max_ms": 82.039, "p95_ms": 69.431}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.21, "count": 175, "max_ms": 97.199, "p95_ms": 88.3}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.956, "count": 175, "max_ms": 195.496, "p95_ms": 183.875}, "read_interval": {"avg_ms": 171.738, "count": 174, "max_ms": 205.956, "p95_ms": 194.265}, "triton_wait": {"avg_ms": 65.427, "count": 175, "max_ms": 97.211, "p95_ms": 89.992}, "visualize": {"avg_ms": 5.179, "count": 175, "max_ms": 5.978, "p95_ms": 5.592}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11881}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10853.812, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 470, "h264_decode_error_count": 1400, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:40:09+09:00", "last_error_summary": "[h264 @ 0x5572ddca8580] error while decoding MB 8 7, bytestream -7", "last_exit_at": "2026-07-01T09:40:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.952, "last_started_at": "2026-07-01T09:37:57+09:00", "pid": 93860, "process": {"exists": false, "pid": 93860}, "restart_count": 517, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 518}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.337, "sample_count": 1744, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1425750, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:40:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:40:09+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T09:40:09+09:00", "mjpeg": {"age_sec": 0.156, "exists": true, "mtime": "2026-07-01T09:40:09.755137+09:00", "mtime_epoch": 1782866409.755137, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157423}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:40:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5045.438, "name": "celery", "pid": 267, "rss_mb": 5045.438, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.858, "pushed": 5.858, "read": 5.824}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.466, "count": 175, "max_ms": 38.685, "p95_ms": 36.704}, "gstreamer_push": {"avg_ms": 1.006, "count": 175, "max_ms": 1.457, "p95_ms": 1.066}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.74, "count": 175, "max_ms": 43.769, "p95_ms": 39.004}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.299, "count": 175, "max_ms": 93.45, "p95_ms": 89.835}, "yolo11_640_fire_11class": {"avg_ms": 50.585, "count": 175, "max_ms": 82.039, "p95_ms": 69.431}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.21, "count": 175, "max_ms": 97.199, "p95_ms": 88.3}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.956, "count": 175, "max_ms": 195.496, "p95_ms": 183.875}, "read_interval": {"avg_ms": 171.738, "count": 174, "max_ms": 205.956, "p95_ms": 194.265}, "triton_wait": {"avg_ms": 65.427, "count": 175, "max_ms": 97.211, "p95_ms": 89.992}, "visualize": {"avg_ms": 5.179, "count": 175, "max_ms": 5.978, "p95_ms": 5.592}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.875, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11881}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10961.523, "limit_mb": 8796093022207.996}, "detail": {"pid": 93967, "restart_count": 518, "start_count": 519}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 470, "h264_decode_error_count": 1400, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:40:09+09:00", "last_error_summary": "[h264 @ 0x5572ddca8580] error while decoding MB 8 7, bytestream -7", "last_exit_at": "2026-07-01T09:40:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:40:12+09:00", "pid": 93967, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.293, "name": "ffmpeg", "pid": 93967, "rss_mb": 139.293, "state": "S (sleeping)", "threads": 106}, "restart_count": 518, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 519}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.17, "sample_count": 1716, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1425750, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:40:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.867, "last_frame_pushed_at": "2026-07-01T09:40:09+09:00", "last_frame_read_age_sec": 5.026, "last_frame_read_at": "2026-07-01T09:40:09+09:00", "mjpeg": {"age_sec": 4.851, "exists": true, "mtime": "2026-07-01T09:40:09.919140+09:00", "mtime_epoch": 1782866409.9191399, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157432}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:40:14+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5045.438, "name": "celery", "pid": 267, "rss_mb": 5045.438, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.888, "pushed": 4.888, "read": 4.854}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.562, "count": 146, "max_ms": 37.719, "p95_ms": 36.704}, "gstreamer_push": {"avg_ms": 1.006, "count": 146, "max_ms": 1.457, "p95_ms": 1.066}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.671, "count": 146, "max_ms": 43.769, "p95_ms": 39.004}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.745, "count": 146, "max_ms": 93.45, "p95_ms": 89.87}, "yolo11_640_fire_11class": {"avg_ms": 50.646, "count": 146, "max_ms": 82.039, "p95_ms": 69.431}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.687, "count": 146, "max_ms": 97.199, "p95_ms": 88.337}}, "truncated_count": 0}, "processing_total": {"avg_ms": 161.608, "count": 146, "max_ms": 195.496, "p95_ms": 185.444}, "read_interval": {"avg_ms": 172.401, "count": 145, "max_ms": 205.956, "p95_ms": 195.881}, "triton_wait": {"avg_ms": 65.921, "count": 146, "max_ms": 97.211, "p95_ms": 90.14}, "visualize": {"avg_ms": 5.182, "count": 146, "max_ms": 5.978, "p95_ms": 5.592}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10791.414, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x558510472a40] error while decoding MB 15 52, bytestream -37 [in#0/rtsp @ 0x5585103ffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585103ffe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 207, "h264_decode_error_count": 213, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:41:35+09:00", "last_error_summary": "[h264 @ 0x558510472a40] error while decoding MB 15 52, bytestream -37 [in#0/rtsp @ 0x5585103ffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585103ffe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:36:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 331.42, "last_started_at": "2026-07-01T09:36:04+09:00", "pid": 93648, "process": {"exists": false, "pid": 93648}, "restart_count": 206, "restart_count_per_hour": 4, "rtsp_timeout_count": 340, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 207}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.601, "sample_count": 2564, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1616054, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:41:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-01T09:41:35+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-01T09:41:35+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-01T09:41:35.636570+09:00", "mtime_epoch": 1782866495.63657, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220661}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:41:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1822.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 101, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.372, "pushed": 3.372, "read": 3.372}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.117, "count": 101, "max_ms": 33.738, "p95_ms": 30.493}, "gstreamer_push": {"avg_ms": 1.195, "count": 101, "max_ms": 1.648, "p95_ms": 1.372}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.905, "count": 101, "max_ms": 63.341, "p95_ms": 57.058}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.418, "count": 101, "max_ms": 111.014, "p95_ms": 86.749}, "yolo11_640_fire_11class": {"avg_ms": 54.694, "count": 101, "max_ms": 69.876, "p95_ms": 67.228}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.333, "count": 101, "max_ms": 107.133, "p95_ms": 84.958}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.573, "count": 101, "max_ms": 139.815, "p95_ms": 116.956}, "read_interval": {"avg_ms": 296.779, "count": 101, "max_ms": 10163.609, "p95_ms": 133.794}, "triton_wait": {"avg_ms": 63.47, "count": 101, "max_ms": 111.027, "p95_ms": 86.78}, "visualize": {"avg_ms": 3.373, "count": 101, "max_ms": 4.457, "p95_ms": 3.75}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10791.359, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:41:35+09:00", "last_error_summary": "[h264 @ 0x558510472a40] error while decoding MB 15 52, bytestream -37 [in#0/rtsp @ 0x5585103ffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585103ffe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 207, "h264_decode_error_count": 213, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:41:35+09:00", "last_error_summary": "[h264 @ 0x558510472a40] error while decoding MB 15 52, bytestream -37 [in#0/rtsp @ 0x5585103ffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585103ffe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:41:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 331.464, "last_started_at": "2026-07-01T09:36:04+09:00", "pid": 93648, "process": {"exists": false, "pid": 93648}, "restart_count": 206, "restart_count_per_hour": 4, "rtsp_timeout_count": 340, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 207}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.601, "sample_count": 2565, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1616055, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:41:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T09:41:35+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T09:41:35+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-01T09:41:35.636570+09:00", "mtime_epoch": 1782866495.63657, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220661}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:41:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1822.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.401, "pushed": 3.401, "read": 3.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.117, "count": 102, "max_ms": 33.738, "p95_ms": 30.493}, "gstreamer_push": {"avg_ms": 1.195, "count": 102, "max_ms": 1.648, "p95_ms": 1.372}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.88, "count": 102, "max_ms": 63.341, "p95_ms": 57.058}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.441, "count": 102, "max_ms": 111.014, "p95_ms": 86.749}, "yolo11_640_fire_11class": {"avg_ms": 54.803, "count": 102, "max_ms": 69.876, "p95_ms": 67.228}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.386, "count": 102, "max_ms": 107.133, "p95_ms": 84.958}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.605, "count": 102, "max_ms": 139.815, "p95_ms": 116.956}, "read_interval": {"avg_ms": 296.779, "count": 101, "max_ms": 10163.609, "p95_ms": 133.794}, "triton_wait": {"avg_ms": 63.492, "count": 102, "max_ms": 111.027, "p95_ms": 86.78}, "visualize": {"avg_ms": 3.383, "count": 102, "max_ms": 4.457, "p95_ms": 3.808}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10791.184, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 207, "h264_decode_error_count": 213, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:41:35+09:00", "last_error_summary": "[h264 @ 0x558510472a40] error while decoding MB 15 52, bytestream -37 [in#0/rtsp @ 0x5585103ffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585103ffe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:41:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 331.472, "last_started_at": "2026-07-01T09:36:04+09:00", "pid": 93648, "process": {"exists": false, "pid": 93648}, "restart_count": 206, "restart_count_per_hour": 4, "rtsp_timeout_count": 340, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 207}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.601, "sample_count": 2565, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1616055, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:41:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T09:41:35+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-01T09:41:35+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T09:41:35.636570+09:00", "mtime_epoch": 1782866495.63657, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220661}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:41:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1822.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.4, "pushed": 3.4, "read": 3.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.1, "count": 101, "max_ms": 33.738, "p95_ms": 30.493}, "gstreamer_push": {"avg_ms": 1.195, "count": 102, "max_ms": 1.648, "p95_ms": 1.372}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.953, "count": 101, "max_ms": 63.341, "p95_ms": 57.058}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.529, "count": 101, "max_ms": 111.014, "p95_ms": 86.749}, "yolo11_640_fire_11class": {"avg_ms": 54.886, "count": 101, "max_ms": 69.876, "p95_ms": 67.228}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.475, "count": 101, "max_ms": 107.133, "p95_ms": 84.958}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.605, "count": 102, "max_ms": 139.815, "p95_ms": 116.956}, "read_interval": {"avg_ms": 296.779, "count": 101, "max_ms": 10163.609, "p95_ms": 133.794}, "triton_wait": {"avg_ms": 63.581, "count": 101, "max_ms": 111.027, "p95_ms": 86.78}, "visualize": {"avg_ms": 3.379, "count": 101, "max_ms": 4.457, "p95_ms": 3.75}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10812.168, "limit_mb": 8796093022207.996}, "detail": {"pid": 94074, "restart_count": 207, "start_count": 208}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 207, "h264_decode_error_count": 213, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:41:35+09:00", "last_error_summary": "[h264 @ 0x558510472a40] error while decoding MB 15 52, bytestream -37 [in#0/rtsp @ 0x5585103ffe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5585103ffe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:41:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:41:37+09:00", "pid": 94074, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.547, "name": "ffmpeg", "pid": 94074, "rss_mb": 58.547, "state": "S (sleeping)", "threads": 1}, "restart_count": 207, "restart_count_per_hour": 5, "rtsp_timeout_count": 340, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 208}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 128.565, "sample_count": 2532, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1616055, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:41:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.122, "last_frame_pushed_at": "2026-07-01T09:41:35+09:00", "last_frame_read_age_sec": 4.22, "last_frame_read_at": "2026-07-01T09:41:35+09:00", "mjpeg": {"age_sec": 4.106, "exists": true, "mtime": "2026-07-01T09:41:35.748572+09:00", "mtime_epoch": 1782866495.7485719, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220164}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:41:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1822.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.139, "pushed": 2.139, "read": 2.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.669, "count": 64, "max_ms": 33.738, "p95_ms": 27.462}, "gstreamer_push": {"avg_ms": 1.186, "count": 64, "max_ms": 1.417, "p95_ms": 1.375}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.078, "count": 64, "max_ms": 63.341, "p95_ms": 55.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.657, "count": 64, "max_ms": 111.014, "p95_ms": 86.749}, "yolo11_640_fire_11class": {"avg_ms": 55.043, "count": 64, "max_ms": 68.849, "p95_ms": 67.228}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.657, "count": 64, "max_ms": 107.133, "p95_ms": 84.742}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.319, "count": 64, "max_ms": 139.815, "p95_ms": 118.199}, "read_interval": {"avg_ms": 409.609, "count": 63, "max_ms": 10163.609, "p95_ms": 140.876}, "triton_wait": {"avg_ms": 63.731, "count": 64, "max_ms": 111.027, "p95_ms": 86.78}, "visualize": {"avg_ms": 3.344, "count": 64, "max_ms": 4.418, "p95_ms": 3.666}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11887}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10851.988, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a2d22764c0] error while decoding MB 28 23, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 471, "h264_decode_error_count": 1401, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:42:23+09:00", "last_error_summary": "[h264 @ 0x55a2d22764c0] error while decoding MB 28 23, bytestream -7", "last_exit_at": "2026-07-01T09:40:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.555, "last_started_at": "2026-07-01T09:40:12+09:00", "pid": 93967, "process": {"exists": false, "pid": 93967}, "restart_count": 518, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 519}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 192.954, "sample_count": 1674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1426477, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:42:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T09:42:23+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T09:42:23+09:00", "mjpeg": {"age_sec": 0.032, "exists": true, "mtime": "2026-07-01T09:42:23.293368+09:00", "mtime_epoch": 1782866543.293368, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 154946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:42:23+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5045.953, "name": "celery", "pid": 267, "rss_mb": 5045.953, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 169, "pushed_frames": 169, "read_frames": 169, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.642, "pushed": 5.642, "read": 5.642}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.243, "count": 168, "max_ms": 50.183, "p95_ms": 40.856}, "gstreamer_push": {"avg_ms": 1.0, "count": 169, "max_ms": 1.361, "p95_ms": 1.085}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.451, "count": 168, "max_ms": 67.869, "p95_ms": 42.38}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.48, "count": 168, "max_ms": 109.509, "p95_ms": 95.734}, "yolo11_640_fire_11class": {"avg_ms": 48.707, "count": 168, "max_ms": 76.231, "p95_ms": 66.034}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.607, "count": 168, "max_ms": 122.698, "p95_ms": 90.514}}, "truncated_count": 0}, "processing_total": {"avg_ms": 167.009, "count": 169, "max_ms": 215.647, "p95_ms": 192.457}, "read_interval": {"avg_ms": 177.883, "count": 169, "max_ms": 225.98, "p95_ms": 203.282}, "triton_wait": {"avg_ms": 69.699, "count": 168, "max_ms": 122.711, "p95_ms": 95.746}, "visualize": {"avg_ms": 5.268, "count": 168, "max_ms": 7.552, "p95_ms": 5.913}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11887}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10850.973, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:42:23+09:00", "last_error_summary": "[h264 @ 0x55a2d22764c0] error while decoding MB 28 23, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 471, "h264_decode_error_count": 1401, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:42:23+09:00", "last_error_summary": "[h264 @ 0x55a2d22764c0] error while decoding MB 28 23, bytestream -7", "last_exit_at": "2026-07-01T09:42:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.685, "last_started_at": "2026-07-01T09:40:12+09:00", "pid": 93967, "process": {"exists": false, "pid": 93967}, "restart_count": 518, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 519}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 192.954, "sample_count": 1674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1426478, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:42:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T09:42:23+09:00", "last_frame_read_age_sec": 0.18, "last_frame_read_at": "2026-07-01T09:42:23+09:00", "mjpeg": {"age_sec": 0.161, "exists": true, "mtime": "2026-07-01T09:42:23.293368+09:00", "mtime_epoch": 1782866543.293368, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 154946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:42:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5045.953, "name": "celery", "pid": 267, "rss_mb": 5045.953, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 169, "pushed_frames": 169, "read_frames": 168, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.656, "pushed": 5.656, "read": 5.623}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.31, "count": 169, "max_ms": 50.183, "p95_ms": 40.881}, "gstreamer_push": {"avg_ms": 1.0, "count": 169, "max_ms": 1.361, "p95_ms": 1.085}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.418, "count": 169, "max_ms": 67.869, "p95_ms": 42.38}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.423, "count": 169, "max_ms": 109.509, "p95_ms": 95.734}, "yolo11_640_fire_11class": {"avg_ms": 48.68, "count": 169, "max_ms": 76.231, "p95_ms": 66.034}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.567, "count": 169, "max_ms": 122.698, "p95_ms": 90.514}}, "truncated_count": 0}, "processing_total": {"avg_ms": 167.106, "count": 169, "max_ms": 215.647, "p95_ms": 192.457}, "read_interval": {"avg_ms": 177.937, "count": 168, "max_ms": 225.98, "p95_ms": 203.282}, "triton_wait": {"avg_ms": 69.641, "count": 169, "max_ms": 122.711, "p95_ms": 95.746}, "visualize": {"avg_ms": 5.272, "count": 169, "max_ms": 7.552, "p95_ms": 5.913}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11887}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10851.09, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 471, "h264_decode_error_count": 1401, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:42:23+09:00", "last_error_summary": "[h264 @ 0x55a2d22764c0] error while decoding MB 28 23, bytestream -7", "last_exit_at": "2026-07-01T09:42:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.689, "last_started_at": "2026-07-01T09:40:12+09:00", "pid": 93967, "process": {"exists": false, "pid": 93967}, "restart_count": 518, "restart_count_per_hour": 23, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 519}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 192.954, "sample_count": 1674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1426478, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:42:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:42:23+09:00", "last_frame_read_age_sec": 0.184, "last_frame_read_at": "2026-07-01T09:42:23+09:00", "mjpeg": {"age_sec": 0.165, "exists": true, "mtime": "2026-07-01T09:42:23.293368+09:00", "mtime_epoch": 1782866543.293368, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 154946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:42:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5045.953, "name": "celery", "pid": 267, "rss_mb": 5045.953, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 169, "pushed_frames": 169, "read_frames": 168, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.656, "pushed": 5.656, "read": 5.622}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.31, "count": 169, "max_ms": 50.183, "p95_ms": 40.881}, "gstreamer_push": {"avg_ms": 1.0, "count": 169, "max_ms": 1.361, "p95_ms": 1.085}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.418, "count": 169, "max_ms": 67.869, "p95_ms": 42.38}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.423, "count": 169, "max_ms": 109.509, "p95_ms": 95.734}, "yolo11_640_fire_11class": {"avg_ms": 48.68, "count": 169, "max_ms": 76.231, "p95_ms": 66.034}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.567, "count": 169, "max_ms": 122.698, "p95_ms": 90.514}}, "truncated_count": 0}, "processing_total": {"avg_ms": 167.106, "count": 169, "max_ms": 215.647, "p95_ms": 192.457}, "read_interval": {"avg_ms": 177.937, "count": 168, "max_ms": 225.98, "p95_ms": 203.282}, "triton_wait": {"avg_ms": 69.641, "count": 169, "max_ms": 122.711, "p95_ms": 95.746}, "visualize": {"avg_ms": 5.272, "count": 169, "max_ms": 7.552, "p95_ms": 5.913}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11887}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10956.777, "limit_mb": 8796093022207.996}, "detail": {"pid": 94226, "restart_count": 519, "start_count": 520}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 471, "h264_decode_error_count": 1401, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:42:23+09:00", "last_error_summary": "[h264 @ 0x55a2d22764c0] error while decoding MB 28 23, bytestream -7", "last_exit_at": "2026-07-01T09:42:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:42:25+09:00", "pid": 94226, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.809, "name": "ffmpeg", "pid": 94226, "rss_mb": 136.809, "state": "S (sleeping)", "threads": 106}, "restart_count": 519, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 520}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.043, "sample_count": 1648, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1426478, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:42:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.3, "last_frame_pushed_at": "2026-07-01T09:42:23+09:00", "last_frame_read_age_sec": 4.476, "last_frame_read_at": "2026-07-01T09:42:23+09:00", "mjpeg": {"age_sec": 4.285, "exists": true, "mtime": "2026-07-01T09:42:23.465371+09:00", "mtime_epoch": 1782866543.465371, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153131}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:42:27+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5045.953, "name": "celery", "pid": 267, "rss_mb": 5045.953, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.887, "pushed": 4.887, "read": 4.854}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.439, "count": 146, "max_ms": 50.183, "p95_ms": 40.881}, "gstreamer_push": {"avg_ms": 1.002, "count": 146, "max_ms": 1.361, "p95_ms": 1.097}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.575, "count": 146, "max_ms": 67.869, "p95_ms": 43.218}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.15, "count": 146, "max_ms": 109.509, "p95_ms": 93.927}, "yolo11_640_fire_11class": {"avg_ms": 48.171, "count": 146, "max_ms": 70.068, "p95_ms": 66.034}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.13, "count": 146, "max_ms": 101.389, "p95_ms": 88.561}}, "truncated_count": 0}, "processing_total": {"avg_ms": 165.778, "count": 146, "max_ms": 211.806, "p95_ms": 190.54}, "read_interval": {"avg_ms": 176.569, "count": 145, "max_ms": 222.158, "p95_ms": 201.412}, "triton_wait": {"avg_ms": 68.2, "count": 146, "max_ms": 109.521, "p95_ms": 93.941}, "visualize": {"avg_ms": 5.253, "count": 146, "max_ms": 7.552, "p95_ms": 5.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11893}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10853.312, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5650f541e100] cabac decode of qscale diff failed at 31 28 [h264 @ 0x5650f541e100] error while decoding MB 31 28, bytestream -3", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 472, "h264_decode_error_count": 1403, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:44:37+09:00", "last_error_summary": "[h264 @ 0x5650f541e100] cabac decode of qscale diff failed at 31 28 [h264 @ 0x5650f541e100] error while decoding MB 31 28, bytestream -3", "last_exit_at": "2026-07-01T09:42:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.849, "last_started_at": "2026-07-01T09:42:25+09:00", "pid": 94226, "process": {"exists": false, "pid": 94226}, "restart_count": 519, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 520}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.008, "sample_count": 1689, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1427250, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:44:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.054, "last_frame_pushed_at": "2026-07-01T09:44:37+09:00", "last_frame_read_age_sec": 0.046, "last_frame_read_at": "2026-07-01T09:44:37+09:00", "mjpeg": {"age_sec": 0.035, "exists": true, "mtime": "2026-07-01T09:44:37.563627+09:00", "mtime_epoch": 1782866677.563627, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157269}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:44:37+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5046.211, "name": "celery", "pid": 267, "rss_mb": 5046.211, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 182, "pushed_frames": 182, "read_frames": 182, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.096, "pushed": 6.096, "read": 6.096}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.749, "count": 182, "max_ms": 50.721, "p95_ms": 37.318}, "gstreamer_push": {"avg_ms": 0.798, "count": 182, "max_ms": 1.04, "p95_ms": 1.007}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.969, "count": 182, "max_ms": 39.952, "p95_ms": 35.948}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.95, "count": 182, "max_ms": 91.165, "p95_ms": 80.49}, "yolo11_640_fire_11class": {"avg_ms": 48.022, "count": 182, "max_ms": 67.399, "p95_ms": 60.223}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.027, "count": 182, "max_ms": 90.165, "p95_ms": 78.785}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.642, "count": 182, "max_ms": 185.895, "p95_ms": 174.243}, "read_interval": {"avg_ms": 164.695, "count": 182, "max_ms": 340.483, "p95_ms": 183.906}, "triton_wait": {"avg_ms": 61.497, "count": 182, "max_ms": 91.175, "p95_ms": 81.422}, "visualize": {"avg_ms": 5.059, "count": 182, "max_ms": 6.554, "p95_ms": 5.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.856, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11893}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10852.406, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:44:37+09:00", "last_error_summary": "[h264 @ 0x5650f541e100] cabac decode of qscale diff failed at 31 28 [h264 @ 0x5650f541e100] error while decoding MB 31 28, bytestream -3", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 472, "h264_decode_error_count": 1403, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:44:37+09:00", "last_error_summary": "[h264 @ 0x5650f541e100] cabac decode of qscale diff failed at 31 28 [h264 @ 0x5650f541e100] error while decoding MB 31 28, bytestream -3", "last_exit_at": "2026-07-01T09:44:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.953, "last_started_at": "2026-07-01T09:42:25+09:00", "pid": 94226, "process": {"exists": false, "pid": 94226}, "restart_count": 519, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 520}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.008, "sample_count": 1689, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1427251, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:44:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:44:37+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-01T09:44:37+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-01T09:44:37.563627+09:00", "mtime_epoch": 1782866677.563627, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157269}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:44:37+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5046.211, "name": "celery", "pid": 267, "rss_mb": 5046.211, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 182, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.108, "pushed": 6.108, "read": 6.075}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.723, "count": 182, "max_ms": 50.721, "p95_ms": 37.318}, "gstreamer_push": {"avg_ms": 0.797, "count": 183, "max_ms": 1.04, "p95_ms": 1.007}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.977, "count": 182, "max_ms": 39.952, "p95_ms": 35.948}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.911, "count": 182, "max_ms": 91.165, "p95_ms": 80.49}, "yolo11_640_fire_11class": {"avg_ms": 48.019, "count": 182, "max_ms": 67.399, "p95_ms": 60.223}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.984, "count": 182, "max_ms": 90.165, "p95_ms": 78.785}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.599, "count": 183, "max_ms": 185.895, "p95_ms": 174.243}, "read_interval": {"avg_ms": 164.695, "count": 182, "max_ms": 340.483, "p95_ms": 183.906}, "triton_wait": {"avg_ms": 61.453, "count": 182, "max_ms": 91.175, "p95_ms": 81.422}, "visualize": {"avg_ms": 5.055, "count": 182, "max_ms": 6.554, "p95_ms": 5.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11893}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10852.398, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 472, "h264_decode_error_count": 1403, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:44:37+09:00", "last_error_summary": "[h264 @ 0x5650f541e100] cabac decode of qscale diff failed at 31 28 [h264 @ 0x5650f541e100] error while decoding MB 31 28, bytestream -3", "last_exit_at": "2026-07-01T09:44:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.957, "last_started_at": "2026-07-01T09:42:25+09:00", "pid": 94226, "process": {"exists": false, "pid": 94226}, "restart_count": 519, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 520}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.008, "sample_count": 1689, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1427251, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:44:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T09:44:37+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-01T09:44:37+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-01T09:44:37.563627+09:00", "mtime_epoch": 1782866677.563627, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157269}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:44:37+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5046.211, "name": "celery", "pid": 267, "rss_mb": 5046.211, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 182, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.107, "pushed": 6.107, "read": 6.074}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.723, "count": 182, "max_ms": 50.721, "p95_ms": 37.318}, "gstreamer_push": {"avg_ms": 0.797, "count": 183, "max_ms": 1.04, "p95_ms": 1.007}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.977, "count": 182, "max_ms": 39.952, "p95_ms": 35.948}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.911, "count": 182, "max_ms": 91.165, "p95_ms": 80.49}, "yolo11_640_fire_11class": {"avg_ms": 48.019, "count": 182, "max_ms": 67.399, "p95_ms": 60.223}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.984, "count": 182, "max_ms": 90.165, "p95_ms": 78.785}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.599, "count": 183, "max_ms": 185.895, "p95_ms": 174.243}, "read_interval": {"avg_ms": 164.695, "count": 182, "max_ms": 340.483, "p95_ms": 183.906}, "triton_wait": {"avg_ms": 61.453, "count": 182, "max_ms": 91.175, "p95_ms": 81.422}, "visualize": {"avg_ms": 5.055, "count": 182, "max_ms": 6.554, "p95_ms": 5.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11893}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10963.492, "limit_mb": 8796093022207.996}, "detail": {"pid": 94333, "restart_count": 520, "start_count": 521}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 472, "h264_decode_error_count": 1403, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:44:37+09:00", "last_error_summary": "[h264 @ 0x5650f541e100] cabac decode of qscale diff failed at 31 28 [h264 @ 0x5650f541e100] error while decoding MB 31 28, bytestream -3", "last_exit_at": "2026-07-01T09:44:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:44:40+09:00", "pid": 94333, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.324, "name": "ffmpeg", "pid": 94333, "rss_mb": 142.324, "state": "S (sleeping)", "threads": 106}, "restart_count": 520, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 521}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 189.046, "sample_count": 1662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1427251, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:44:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.533, "last_frame_pushed_at": "2026-07-01T09:44:37+09:00", "last_frame_read_age_sec": 4.68, "last_frame_read_at": "2026-07-01T09:44:37+09:00", "mjpeg": {"age_sec": 4.517, "exists": true, "mtime": "2026-07-01T09:44:37.715630+09:00", "mtime_epoch": 1782866677.7156296, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157278}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:44:42+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5046.211, "name": "celery", "pid": 267, "rss_mb": 5046.211, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 157, "pushed_frames": 157, "read_frames": 156, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.234, "pushed": 5.234, "read": 5.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.482, "count": 156, "max_ms": 50.721, "p95_ms": 38.652}, "gstreamer_push": {"avg_ms": 0.764, "count": 157, "max_ms": 0.985, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.047, "count": 156, "max_ms": 39.952, "p95_ms": 36.345}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.062, "count": 156, "max_ms": 83.884, "p95_ms": 80.49}, "yolo11_640_fire_11class": {"avg_ms": 47.932, "count": 156, "max_ms": 67.399, "p95_ms": 60.223}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.219, "count": 156, "max_ms": 85.17, "p95_ms": 78.738}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.104, "count": 157, "max_ms": 183.962, "p95_ms": 173.0}, "read_interval": {"avg_ms": 163.269, "count": 156, "max_ms": 340.483, "p95_ms": 183.816}, "triton_wait": {"avg_ms": 60.631, "count": 156, "max_ms": 85.182, "p95_ms": 81.422}, "visualize": {"avg_ms": 5.032, "count": 156, "max_ms": 6.554, "p95_ms": 5.669}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11900}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10854.836, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564cfdb18200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564cfdb18200] error while decoding MB 0 22, bytestream 65683 [h264 @ 0x564cfdbe7080] error while decoding MB 67 37, bytestream -19", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 473, "h264_decode_error_count": 1405, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:47:02+09:00", "last_error_summary": "[h264 @ 0x564cfdb18200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564cfdb18200] error while decoding MB 0 22, bytestream 65683 [h264 @ 0x564cfdbe7080] error while decoding MB 67 37, bytestream -19", "last_exit_at": "2026-07-01T09:44:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 142.654, "last_started_at": "2026-07-01T09:44:40+09:00", "pid": 94333, "process": {"exists": false, "pid": 94333}, "restart_count": 520, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 521}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.254, "sample_count": 1690, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1428051, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:47:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T09:47:02+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T09:47:02+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T09:47:02.842084+09:00", "mtime_epoch": 1782866822.8420844, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153439}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:47:02+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5046.727, "name": "celery", "pid": 267, "rss_mb": 5046.727, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 175, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.847, "pushed": 5.847, "read": 5.847}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.233, "count": 175, "max_ms": 41.906, "p95_ms": 40.342}, "gstreamer_push": {"avg_ms": 0.985, "count": 175, "max_ms": 1.362, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.547, "count": 175, "max_ms": 47.376, "p95_ms": 37.377}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.521, "count": 175, "max_ms": 94.827, "p95_ms": 86.511}, "yolo11_640_fire_11class": {"avg_ms": 48.346, "count": 175, "max_ms": 71.733, "p95_ms": 61.078}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.588, "count": 175, "max_ms": 91.014, "p95_ms": 82.503}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.856, "count": 175, "max_ms": 203.565, "p95_ms": 187.097}, "read_interval": {"avg_ms": 171.648, "count": 175, "max_ms": 213.287, "p95_ms": 197.557}, "triton_wait": {"avg_ms": 65.739, "count": 175, "max_ms": 94.841, "p95_ms": 86.523}, "visualize": {"avg_ms": 5.201, "count": 175, "max_ms": 7.807, "p95_ms": 6.6}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11900}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10853.809, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:47:02+09:00", "last_error_summary": "[h264 @ 0x564cfdb18200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564cfdb18200] error while decoding MB 0 22, bytestream 65683 [h264 @ 0x564cfdbe7080] error while decoding MB 67 37, bytestream -19", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 473, "h264_decode_error_count": 1405, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:47:02+09:00", "last_error_summary": "[h264 @ 0x564cfdb18200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564cfdb18200] error while decoding MB 0 22, bytestream 65683 [h264 @ 0x564cfdbe7080] error while decoding MB 67 37, bytestream -19", "last_exit_at": "2026-07-01T09:47:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 142.769, "last_started_at": "2026-07-01T09:44:40+09:00", "pid": 94333, "process": {"exists": false, "pid": 94333}, "restart_count": 520, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 521}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.254, "sample_count": 1691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1428052, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:47:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:47:02+09:00", "last_frame_read_age_sec": 0.166, "last_frame_read_at": "2026-07-01T09:47:02+09:00", "mjpeg": {"age_sec": 0.158, "exists": true, "mtime": "2026-07-01T09:47:02.842084+09:00", "mtime_epoch": 1782866822.8420844, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153439}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:47:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5046.727, "name": "celery", "pid": 267, "rss_mb": 5046.727, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.863, "pushed": 5.863, "read": 5.83}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.234, "count": 175, "max_ms": 41.906, "p95_ms": 40.342}, "gstreamer_push": {"avg_ms": 0.986, "count": 175, "max_ms": 1.362, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.56, "count": 175, "max_ms": 47.376, "p95_ms": 37.377}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.484, "count": 175, "max_ms": 94.827, "p95_ms": 86.511}, "yolo11_640_fire_11class": {"avg_ms": 48.347, "count": 175, "max_ms": 71.733, "p95_ms": 61.078}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.582, "count": 175, "max_ms": 91.014, "p95_ms": 82.503}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.85, "count": 175, "max_ms": 203.565, "p95_ms": 187.097}, "read_interval": {"avg_ms": 171.646, "count": 174, "max_ms": 213.287, "p95_ms": 197.557}, "triton_wait": {"avg_ms": 65.719, "count": 175, "max_ms": 94.841, "p95_ms": 86.523}, "visualize": {"avg_ms": 5.204, "count": 175, "max_ms": 7.807, "p95_ms": 6.6}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11900}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10853.801, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 473, "h264_decode_error_count": 1405, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:47:02+09:00", "last_error_summary": "[h264 @ 0x564cfdb18200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564cfdb18200] error while decoding MB 0 22, bytestream 65683 [h264 @ 0x564cfdbe7080] error while decoding MB 67 37, bytestream -19", "last_exit_at": "2026-07-01T09:47:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 142.774, "last_started_at": "2026-07-01T09:44:40+09:00", "pid": 94333, "process": {"exists": false, "pid": 94333}, "restart_count": 520, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 521}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 188.254, "sample_count": 1691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1428052, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:47:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:47:02+09:00", "last_frame_read_age_sec": 0.17, "last_frame_read_at": "2026-07-01T09:47:02+09:00", "mjpeg": {"age_sec": 0.163, "exists": true, "mtime": "2026-07-01T09:47:02.842084+09:00", "mtime_epoch": 1782866822.8420844, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153439}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:47:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5046.727, "name": "celery", "pid": 267, "rss_mb": 5046.727, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 175, "pushed_frames": 175, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.863, "pushed": 5.863, "read": 5.829}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.234, "count": 175, "max_ms": 41.906, "p95_ms": 40.342}, "gstreamer_push": {"avg_ms": 0.986, "count": 175, "max_ms": 1.362, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.56, "count": 175, "max_ms": 47.376, "p95_ms": 37.377}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.484, "count": 175, "max_ms": 94.827, "p95_ms": 86.511}, "yolo11_640_fire_11class": {"avg_ms": 48.347, "count": 175, "max_ms": 71.733, "p95_ms": 61.078}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.582, "count": 175, "max_ms": 91.014, "p95_ms": 82.503}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.85, "count": 175, "max_ms": 203.565, "p95_ms": 187.097}, "read_interval": {"avg_ms": 171.646, "count": 174, "max_ms": 213.287, "p95_ms": 197.557}, "triton_wait": {"avg_ms": 65.719, "count": 175, "max_ms": 94.841, "p95_ms": 86.523}, "visualize": {"avg_ms": 5.204, "count": 175, "max_ms": 7.807, "p95_ms": 6.6}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.851, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11900}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10961.5, "limit_mb": 8796093022207.996}, "detail": {"pid": 94440, "restart_count": 521, "start_count": 522}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 473, "h264_decode_error_count": 1405, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:47:02+09:00", "last_error_summary": "[h264 @ 0x564cfdb18200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x564cfdb18200] error while decoding MB 0 22, bytestream 65683 [h264 @ 0x564cfdbe7080] error while decoding MB 67 37, bytestream -19", "last_exit_at": "2026-07-01T09:47:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:47:05+09:00", "pid": 94440, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.375, "name": "ffmpeg", "pid": 94440, "rss_mb": 139.375, "state": "S (sleeping)", "threads": 106}, "restart_count": 521, "restart_count_per_hour": 25, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 522}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.594, "sample_count": 1668, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1428052, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:47:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.128, "last_frame_pushed_at": "2026-07-01T09:47:02+09:00", "last_frame_read_age_sec": 4.29, "last_frame_read_at": "2026-07-01T09:47:02+09:00", "mjpeg": {"age_sec": 4.11, "exists": true, "mtime": "2026-07-01T09:47:03.014087+09:00", "mtime_epoch": 1782866823.0140872, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153871}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:47:07+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5046.727, "name": "celery", "pid": 267, "rss_mb": 5046.727, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 151, "pushed_frames": 151, "read_frames": 150, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.057, "pushed": 5.057, "read": 5.023}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.379, "count": 151, "max_ms": 41.906, "p95_ms": 40.342}, "gstreamer_push": {"avg_ms": 0.983, "count": 151, "max_ms": 1.362, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.514, "count": 151, "max_ms": 47.376, "p95_ms": 37.064}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.387, "count": 151, "max_ms": 94.827, "p95_ms": 83.282}, "yolo11_640_fire_11class": {"avg_ms": 48.737, "count": 151, "max_ms": 71.733, "p95_ms": 61.078}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.46, "count": 151, "max_ms": 91.014, "p95_ms": 81.773}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.806, "count": 151, "max_ms": 196.372, "p95_ms": 187.097}, "read_interval": {"avg_ms": 171.585, "count": 150, "max_ms": 209.476, "p95_ms": 197.557}, "triton_wait": {"avg_ms": 65.658, "count": 151, "max_ms": 94.841, "p95_ms": 84.491}, "visualize": {"avg_ms": 5.23, "count": 151, "max_ms": 7.807, "p95_ms": 6.6}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.861, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11906}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10855.586, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561b081ad8c0] error while decoding MB 26 38, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 474, "h264_decode_error_count": 1406, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:49:15+09:00", "last_error_summary": "[h264 @ 0x561b081ad8c0] error while decoding MB 26 38, bytestream -7", "last_exit_at": "2026-07-01T09:47:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.723, "last_started_at": "2026-07-01T09:47:05+09:00", "pid": 94440, "process": {"exists": false, "pid": 94440}, "restart_count": 521, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 522}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.439, "sample_count": 1662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1428778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:49:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T09:49:15+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T09:49:15+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T09:49:15.803583+09:00", "mtime_epoch": 1782866955.8035831, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158899}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:49:15+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5046.984, "name": "celery", "pid": 267, "rss_mb": 5046.984, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 172, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.749, "pushed": 5.749, "read": 5.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.93, "count": 172, "max_ms": 44.301, "p95_ms": 38.388}, "gstreamer_push": {"avg_ms": 1.018, "count": 172, "max_ms": 1.332, "p95_ms": 1.115}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.308, "count": 172, "max_ms": 44.694, "p95_ms": 38.234}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.121, "count": 172, "max_ms": 114.539, "p95_ms": 94.264}, "yolo11_640_fire_11class": {"avg_ms": 48.88, "count": 172, "max_ms": 71.182, "p95_ms": 64.666}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.797, "count": 172, "max_ms": 103.212, "p95_ms": 84.709}}, "truncated_count": 0}, "processing_total": {"avg_ms": 163.971, "count": 172, "max_ms": 211.801, "p95_ms": 191.455}, "read_interval": {"avg_ms": 174.562, "count": 172, "max_ms": 222.067, "p95_ms": 201.128}, "triton_wait": {"avg_ms": 69.232, "count": 172, "max_ms": 114.549, "p95_ms": 94.278}, "visualize": {"avg_ms": 5.152, "count": 172, "max_ms": 6.068, "p95_ms": 5.695}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11906}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10854.562, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:49:15+09:00", "last_error_summary": "[h264 @ 0x561b081ad8c0] error while decoding MB 26 38, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 474, "h264_decode_error_count": 1406, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:49:15+09:00", "last_error_summary": "[h264 @ 0x561b081ad8c0] error while decoding MB 26 38, bytestream -7", "last_exit_at": "2026-07-01T09:49:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.816, "last_started_at": "2026-07-01T09:47:05+09:00", "pid": 94440, "process": {"exists": false, "pid": 94440}, "restart_count": 521, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 522}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.439, "sample_count": 1662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1428779, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:49:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:49:15+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-01T09:49:15+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-01T09:49:15.803583+09:00", "mtime_epoch": 1782866955.8035831, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158899}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:49:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5046.984, "name": "celery", "pid": 267, "rss_mb": 5046.984, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 171, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.764, "pushed": 5.764, "read": 5.73}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.934, "count": 172, "max_ms": 44.301, "p95_ms": 38.388}, "gstreamer_push": {"avg_ms": 1.019, "count": 172, "max_ms": 1.332, "p95_ms": 1.131}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.289, "count": 172, "max_ms": 44.694, "p95_ms": 38.234}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.131, "count": 172, "max_ms": 114.539, "p95_ms": 94.264}, "yolo11_640_fire_11class": {"avg_ms": 48.893, "count": 172, "max_ms": 71.182, "p95_ms": 64.666}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.823, "count": 172, "max_ms": 103.212, "p95_ms": 84.709}}, "truncated_count": 0}, "processing_total": {"avg_ms": 163.861, "count": 172, "max_ms": 211.801, "p95_ms": 191.455}, "read_interval": {"avg_ms": 174.606, "count": 171, "max_ms": 222.067, "p95_ms": 201.128}, "triton_wait": {"avg_ms": 69.253, "count": 172, "max_ms": 114.549, "p95_ms": 94.278}, "visualize": {"avg_ms": 5.147, "count": 172, "max_ms": 6.068, "p95_ms": 5.695}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.841, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11906}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10854.68, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 474, "h264_decode_error_count": 1406, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:49:15+09:00", "last_error_summary": "[h264 @ 0x561b081ad8c0] error while decoding MB 26 38, bytestream -7", "last_exit_at": "2026-07-01T09:49:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.821, "last_started_at": "2026-07-01T09:47:05+09:00", "pid": 94440, "process": {"exists": false, "pid": 94440}, "restart_count": 521, "restart_count_per_hour": 24, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 522}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.439, "sample_count": 1662, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1428779, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:49:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T09:49:15+09:00", "last_frame_read_age_sec": 0.15, "last_frame_read_at": "2026-07-01T09:49:15+09:00", "mjpeg": {"age_sec": 0.141, "exists": true, "mtime": "2026-07-01T09:49:15.803583+09:00", "mtime_epoch": 1782866955.8035831, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158899}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:49:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5046.984, "name": "celery", "pid": 267, "rss_mb": 5046.984, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 171, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.763, "pushed": 5.763, "read": 5.729}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.934, "count": 172, "max_ms": 44.301, "p95_ms": 38.388}, "gstreamer_push": {"avg_ms": 1.019, "count": 172, "max_ms": 1.332, "p95_ms": 1.131}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.289, "count": 172, "max_ms": 44.694, "p95_ms": 38.234}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.131, "count": 172, "max_ms": 114.539, "p95_ms": 94.264}, "yolo11_640_fire_11class": {"avg_ms": 48.893, "count": 172, "max_ms": 71.182, "p95_ms": 64.666}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.823, "count": 172, "max_ms": 103.212, "p95_ms": 84.709}}, "truncated_count": 0}, "processing_total": {"avg_ms": 163.861, "count": 172, "max_ms": 211.801, "p95_ms": 191.455}, "read_interval": {"avg_ms": 174.606, "count": 171, "max_ms": 222.067, "p95_ms": 201.128}, "triton_wait": {"avg_ms": 69.253, "count": 172, "max_ms": 114.549, "p95_ms": 94.278}, "visualize": {"avg_ms": 5.147, "count": 172, "max_ms": 6.068, "p95_ms": 5.695}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11906}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10961.152, "limit_mb": 8796093022207.996}, "detail": {"pid": 94547, "restart_count": 522, "start_count": 523}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 474, "h264_decode_error_count": 1406, "h264_decode_error_count_per_hour": 42, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:49:15+09:00", "last_error_summary": "[h264 @ 0x561b081ad8c0] error while decoding MB 26 38, bytestream -7", "last_exit_at": "2026-07-01T09:49:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:49:18+09:00", "pid": 94547, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.371, "name": "ffmpeg", "pid": 94547, "rss_mb": 136.371, "state": "S (sleeping)", "threads": 106}, "restart_count": 522, "restart_count_per_hour": 25, "rtsp_timeout_count": 402, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 523}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.557, "sample_count": 1634, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1428779, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:49:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.624, "last_frame_pushed_at": "2026-07-01T09:49:15+09:00", "last_frame_read_age_sec": 4.765, "last_frame_read_at": "2026-07-01T09:49:15+09:00", "mjpeg": {"age_sec": 4.608, "exists": true, "mtime": "2026-07-01T09:49:15.951584+09:00", "mtime_epoch": 1782866955.9515843, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159258}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:49:20+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5046.984, "name": "celery", "pid": 267, "rss_mb": 5046.984, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.902, "pushed": 4.902, "read": 4.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.033, "count": 146, "max_ms": 44.301, "p95_ms": 39.426}, "gstreamer_push": {"avg_ms": 1.025, "count": 147, "max_ms": 1.332, "p95_ms": 1.131}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.16, "count": 146, "max_ms": 44.694, "p95_ms": 37.471}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.556, "count": 146, "max_ms": 114.539, "p95_ms": 90.676}, "yolo11_640_fire_11class": {"avg_ms": 48.604, "count": 146, "max_ms": 67.764, "p95_ms": 63.919}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.197, "count": 146, "max_ms": 103.212, "p95_ms": 81.47}}, "truncated_count": 0}, "processing_total": {"avg_ms": 163.04, "count": 147, "max_ms": 211.801, "p95_ms": 187.341}, "read_interval": {"avg_ms": 173.862, "count": 146, "max_ms": 222.067, "p95_ms": 198.942}, "triton_wait": {"avg_ms": 68.61, "count": 146, "max_ms": 114.549, "p95_ms": 90.687}, "visualize": {"avg_ms": 5.127, "count": 146, "max_ms": 6.052, "p95_ms": 5.629}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13508}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10799.25, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x564e61328000] error while decoding MB 110 60, bytestream -16 [in#0/rtsp @ 0x564e61131e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564e61131e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 208, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:50:56+09:00", "last_error_summary": "[h264 @ 0x564e61328000] error while decoding MB 110 60, bytestream -16 [in#0/rtsp @ 0x564e61131e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564e61131e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:41:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 558.79, "last_started_at": "2026-07-01T09:41:37+09:00", "pid": 94074, "process": {"exists": false, "pid": 94074}, "restart_count": 207, "restart_count_per_hour": 5, "rtsp_timeout_count": 342, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 208}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.87, "sample_count": 2583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1621053, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:50:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T09:50:56+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T09:50:56+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T09:50:56.596501+09:00", "mtime_epoch": 1782867056.5965014, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 216020}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:50:56+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1825.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 107, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.568, "pushed": 3.568, "read": 3.568}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.376, "count": 107, "max_ms": 30.456, "p95_ms": 24.608}, "gstreamer_push": {"avg_ms": 1.193, "count": 107, "max_ms": 1.313, "p95_ms": 1.289}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.741, "count": 107, "max_ms": 59.381, "p95_ms": 53.812}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.784, "count": 107, "max_ms": 86.032, "p95_ms": 80.084}, "yolo11_640_fire_11class": {"avg_ms": 50.637, "count": 107, "max_ms": 68.032, "p95_ms": 62.188}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.284, "count": 107, "max_ms": 83.816, "p95_ms": 78.794}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.276, "count": 107, "max_ms": 114.731, "p95_ms": 109.232}, "read_interval": {"avg_ms": 280.526, "count": 107, "max_ms": 10169.782, "p95_ms": 127.032}, "triton_wait": {"avg_ms": 58.012, "count": 107, "max_ms": 86.044, "p95_ms": 80.95}, "visualize": {"avg_ms": 3.309, "count": 107, "max_ms": 3.882, "p95_ms": 3.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13508}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10798.938, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:50:56+09:00", "last_error_summary": "[h264 @ 0x564e61328000] error while decoding MB 110 60, bytestream -16 [in#0/rtsp @ 0x564e61131e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564e61131e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 208, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:50:56+09:00", "last_error_summary": "[h264 @ 0x564e61328000] error while decoding MB 110 60, bytestream -16 [in#0/rtsp @ 0x564e61131e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564e61131e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:50:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 558.851, "last_started_at": "2026-07-01T09:41:37+09:00", "pid": 94074, "process": {"exists": false, "pid": 94074}, "restart_count": 207, "restart_count_per_hour": 5, "rtsp_timeout_count": 342, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 208}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.87, "sample_count": 2583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1621054, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:50:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:50:56+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T09:50:56+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-01T09:50:56.596501+09:00", "mtime_epoch": 1782867056.5965014, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 216020}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:50:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1825.062, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.576, "pushed": 3.576, "read": 3.542}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.378, "count": 107, "max_ms": 30.456, "p95_ms": 24.608}, "gstreamer_push": {"avg_ms": 1.193, "count": 107, "max_ms": 1.313, "p95_ms": 1.289}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.914, "count": 107, "max_ms": 59.381, "p95_ms": 54.339}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.986, "count": 107, "max_ms": 86.032, "p95_ms": 80.084}, "yolo11_640_fire_11class": {"avg_ms": 50.728, "count": 107, "max_ms": 68.032, "p95_ms": 62.188}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.502, "count": 107, "max_ms": 83.816, "p95_ms": 78.794}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.477, "count": 107, "max_ms": 114.731, "p95_ms": 109.232}, "read_interval": {"avg_ms": 282.266, "count": 106, "max_ms": 10169.782, "p95_ms": 127.032}, "triton_wait": {"avg_ms": 58.214, "count": 107, "max_ms": 86.044, "p95_ms": 80.95}, "visualize": {"avg_ms": 3.307, "count": 107, "max_ms": 3.882, "p95_ms": 3.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13508}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10798.883, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 208, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:50:56+09:00", "last_error_summary": "[h264 @ 0x564e61328000] error while decoding MB 110 60, bytestream -16 [in#0/rtsp @ 0x564e61131e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564e61131e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:50:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 558.858, "last_started_at": "2026-07-01T09:41:37+09:00", "pid": 94074, "process": {"exists": false, "pid": 94074}, "restart_count": 207, "restart_count_per_hour": 5, "rtsp_timeout_count": 342, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 208}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.87, "sample_count": 2583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1621054, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:50:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T09:50:56+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-01T09:50:56+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-01T09:50:56.596501+09:00", "mtime_epoch": 1782867056.5965014, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 216020}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:50:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1825.062, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.575, "pushed": 3.575, "read": 3.541}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.378, "count": 107, "max_ms": 30.456, "p95_ms": 24.608}, "gstreamer_push": {"avg_ms": 1.193, "count": 107, "max_ms": 1.313, "p95_ms": 1.289}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.914, "count": 107, "max_ms": 59.381, "p95_ms": 54.339}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.986, "count": 107, "max_ms": 86.032, "p95_ms": 80.084}, "yolo11_640_fire_11class": {"avg_ms": 50.728, "count": 107, "max_ms": 68.032, "p95_ms": 62.188}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.502, "count": 107, "max_ms": 83.816, "p95_ms": 78.794}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.477, "count": 107, "max_ms": 114.731, "p95_ms": 109.232}, "read_interval": {"avg_ms": 282.266, "count": 106, "max_ms": 10169.782, "p95_ms": 127.032}, "triton_wait": {"avg_ms": 58.214, "count": 107, "max_ms": 86.044, "p95_ms": 80.95}, "visualize": {"avg_ms": 3.307, "count": 107, "max_ms": 3.882, "p95_ms": 3.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13508}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10966.551, "limit_mb": 8796093022207.996}, "detail": {"pid": 94654, "restart_count": 208, "start_count": 209}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 208, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:50:56+09:00", "last_error_summary": "[h264 @ 0x564e61328000] error while decoding MB 110 60, bytestream -16 [in#0/rtsp @ 0x564e61131e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564e61131e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:50:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:50:59+09:00", "pid": 94654, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.117, "name": "ffmpeg", "pid": 94654, "rss_mb": 197.117, "state": "S (sleeping)", "threads": 106}, "restart_count": 208, "restart_count_per_hour": 6, "rtsp_timeout_count": 342, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 209}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.706, "sample_count": 2539, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1621054, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:50:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.915, "last_frame_pushed_at": "2026-07-01T09:50:56+09:00", "last_frame_read_age_sec": 5.02, "last_frame_read_at": "2026-07-01T09:50:56+09:00", "mjpeg": {"age_sec": 4.899, "exists": true, "mtime": "2026-07-01T09:50:56.712502+09:00", "mtime_epoch": 1782867056.7125025, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 215911}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:51:01+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1825.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.972, "pushed": 1.972, "read": 1.938}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.437, "count": 59, "max_ms": 26.815, "p95_ms": 24.608}, "gstreamer_push": {"avg_ms": 1.187, "count": 59, "max_ms": 1.313, "p95_ms": 1.293}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.936, "count": 59, "max_ms": 56.13, "p95_ms": 54.469}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.718, "count": 59, "max_ms": 86.032, "p95_ms": 80.084}, "yolo11_640_fire_11class": {"avg_ms": 51.076, "count": 59, "max_ms": 68.032, "p95_ms": 66.099}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.884, "count": 59, "max_ms": 83.816, "p95_ms": 80.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.38, "count": 59, "max_ms": 114.731, "p95_ms": 111.91}, "read_interval": {"avg_ms": 430.773, "count": 58, "max_ms": 10169.782, "p95_ms": 130.697}, "triton_wait": {"avg_ms": 58.11, "count": 59, "max_ms": 86.044, "p95_ms": 83.775}, "visualize": {"avg_ms": 3.293, "count": 59, "max_ms": 3.882, "p95_ms": 3.544}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10859.742, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55bf168cbc80] left block unavailable for requested intra mode [h264 @ 0x55bf168cbc80] error while decoding MB 0 29, bytestream 6715 [h264 @ 0x55bf15c4de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bf15c4de80] error while decoding MB 0 9, bytestream 7550 [in#0/rtsp @ 0x55bf15bd0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bf15bd0e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 475, "h264_decode_error_count": 1408, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:52:26+09:00", "last_error_summary": "[h264 @ 0x55bf168cbc80] left block unavailable for requested intra mode [h264 @ 0x55bf168cbc80] error while decoding MB 0 29, bytestream 6715 [h264 @ 0x55bf15c4de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bf15c4de80] error while decoding MB 0 9, bytestream 7550 [in#0/rtsp @ 0x55bf15bd0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bf15bd0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:49:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.656, "last_started_at": "2026-07-01T09:49:18+09:00", "pid": 94547, "process": {"exists": false, "pid": 94547}, "restart_count": 522, "restart_count_per_hour": 24, "rtsp_timeout_count": 404, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 523}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 185.863, "sample_count": 1671, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1429829, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:52:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-01T09:52:26+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-01T09:52:26+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-01T09:52:26.165416+09:00", "mtime_epoch": 1782867146.1654165, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160721}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:52:26+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5047.5, "name": "celery", "pid": 267, "rss_mb": 5047.5, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 69, "pushed_frames": 69, "read_frames": 69, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.307, "pushed": 2.307, "read": 2.307}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.926, "count": 69, "max_ms": 36.168, "p95_ms": 35.601}, "gstreamer_push": {"avg_ms": 1.018, "count": 69, "max_ms": 1.165, "p95_ms": 1.141}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.634, "count": 69, "max_ms": 38.496, "p95_ms": 35.493}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.783, "count": 69, "max_ms": 91.779, "p95_ms": 83.326}, "yolo11_640_fire_11class": {"avg_ms": 47.615, "count": 69, "max_ms": 68.354, "p95_ms": 59.651}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.787, "count": 69, "max_ms": 86.43, "p95_ms": 78.729}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.556, "count": 69, "max_ms": 187.723, "p95_ms": 178.936}, "read_interval": {"avg_ms": 435.129, "count": 69, "max_ms": 18868.284, "p95_ms": 192.011}, "triton_wait": {"avg_ms": 62.095, "count": 69, "max_ms": 91.791, "p95_ms": 83.338}, "visualize": {"avg_ms": 5.011, "count": 69, "max_ms": 5.565, "p95_ms": 5.515}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10859.02, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:52:26+09:00", "last_error_summary": "[h264 @ 0x55bf168cbc80] left block unavailable for requested intra mode [h264 @ 0x55bf168cbc80] error while decoding MB 0 29, bytestream 6715 [h264 @ 0x55bf15c4de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bf15c4de80] error while decoding MB 0 9, bytestream 7550 [in#0/rtsp @ 0x55bf15bd0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bf15bd0e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 475, "h264_decode_error_count": 1408, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:52:26+09:00", "last_error_summary": "[h264 @ 0x55bf168cbc80] left block unavailable for requested intra mode [h264 @ 0x55bf168cbc80] error while decoding MB 0 29, bytestream 6715 [h264 @ 0x55bf15c4de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bf15c4de80] error while decoding MB 0 9, bytestream 7550 [in#0/rtsp @ 0x55bf15bd0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bf15bd0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:52:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.74, "last_started_at": "2026-07-01T09:49:18+09:00", "pid": 94547, "process": {"exists": false, "pid": 94547}, "restart_count": 522, "restart_count_per_hour": 24, "rtsp_timeout_count": 404, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 523}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 185.863, "sample_count": 1672, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1429830, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:52:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T09:52:26+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T09:52:26+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-01T09:52:26.165416+09:00", "mtime_epoch": 1782867146.1654165, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160721}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:52:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5047.5, "name": "celery", "pid": 267, "rss_mb": 5047.5, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 70, "pushed_frames": 70, "read_frames": 69, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.334, "pushed": 2.334, "read": 2.3}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.014, "count": 69, "max_ms": 37.228, "p95_ms": 35.732}, "gstreamer_push": {"avg_ms": 1.019, "count": 70, "max_ms": 1.165, "p95_ms": 1.141}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.564, "count": 69, "max_ms": 38.496, "p95_ms": 35.493}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.545, "count": 69, "max_ms": 91.779, "p95_ms": 83.326}, "yolo11_640_fire_11class": {"avg_ms": 47.513, "count": 69, "max_ms": 68.354, "p95_ms": 59.651}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.498, "count": 69, "max_ms": 86.43, "p95_ms": 78.729}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.326, "count": 70, "max_ms": 187.723, "p95_ms": 178.936}, "read_interval": {"avg_ms": 435.129, "count": 69, "max_ms": 18868.284, "p95_ms": 192.011}, "triton_wait": {"avg_ms": 61.858, "count": 69, "max_ms": 91.791, "p95_ms": 83.338}, "visualize": {"avg_ms": 4.999, "count": 69, "max_ms": 5.565, "p95_ms": 5.515}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10859.012, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 475, "h264_decode_error_count": 1408, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:52:26+09:00", "last_error_summary": "[h264 @ 0x55bf168cbc80] left block unavailable for requested intra mode [h264 @ 0x55bf168cbc80] error while decoding MB 0 29, bytestream 6715 [h264 @ 0x55bf15c4de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bf15c4de80] error while decoding MB 0 9, bytestream 7550 [in#0/rtsp @ 0x55bf15bd0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bf15bd0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:52:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 187.745, "last_started_at": "2026-07-01T09:49:18+09:00", "pid": 94547, "process": {"exists": false, "pid": 94547}, "restart_count": 522, "restart_count_per_hour": 24, "rtsp_timeout_count": 404, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 523}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 185.863, "sample_count": 1672, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1429830, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:52:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:52:26+09:00", "last_frame_read_age_sec": 0.146, "last_frame_read_at": "2026-07-01T09:52:26+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-01T09:52:26.165416+09:00", "mtime_epoch": 1782867146.1654165, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160721}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:52:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5047.5, "name": "celery", "pid": 267, "rss_mb": 5047.5, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 70, "pushed_frames": 70, "read_frames": 69, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.333, "pushed": 2.333, "read": 2.3}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.014, "count": 69, "max_ms": 37.228, "p95_ms": 35.732}, "gstreamer_push": {"avg_ms": 1.019, "count": 70, "max_ms": 1.165, "p95_ms": 1.141}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.564, "count": 69, "max_ms": 38.496, "p95_ms": 35.493}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.545, "count": 69, "max_ms": 91.779, "p95_ms": 83.326}, "yolo11_640_fire_11class": {"avg_ms": 47.513, "count": 69, "max_ms": 68.354, "p95_ms": 59.651}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.498, "count": 69, "max_ms": 86.43, "p95_ms": 78.729}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.326, "count": 70, "max_ms": 187.723, "p95_ms": 178.936}, "read_interval": {"avg_ms": 435.129, "count": 69, "max_ms": 18868.284, "p95_ms": 192.011}, "triton_wait": {"avg_ms": 61.858, "count": 69, "max_ms": 91.791, "p95_ms": 83.338}, "visualize": {"avg_ms": 4.999, "count": 69, "max_ms": 5.565, "p95_ms": 5.515}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10967.23, "limit_mb": 8796093022207.996}, "detail": {"pid": 94806, "restart_count": 523, "start_count": 524}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 475, "h264_decode_error_count": 1408, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:52:26+09:00", "last_error_summary": "[h264 @ 0x55bf168cbc80] left block unavailable for requested intra mode [h264 @ 0x55bf168cbc80] error while decoding MB 0 29, bytestream 6715 [h264 @ 0x55bf15c4de80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bf15c4de80] error while decoding MB 0 9, bytestream 7550 [in#0/rtsp @ 0x55bf15bd0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bf15bd0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:52:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:52:28+09:00", "pid": 94806, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.883, "name": "ffmpeg", "pid": 94806, "rss_mb": 138.883, "state": "S (sleeping)", "threads": 106}, "restart_count": 523, "restart_count_per_hour": 25, "rtsp_timeout_count": 404, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 524}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 185.668, "sample_count": 1647, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1429830, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:52:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.26, "last_frame_pushed_at": "2026-07-01T09:52:26+09:00", "last_frame_read_age_sec": 4.398, "last_frame_read_at": "2026-07-01T09:52:26+09:00", "mjpeg": {"age_sec": 4.245, "exists": true, "mtime": "2026-07-01T09:52:26.309418+09:00", "mtime_epoch": 1782867146.309418, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160219}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:52:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5047.5, "name": "celery", "pid": 267, "rss_mb": 5047.5, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.471, "pushed": 1.471, "read": 1.437}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.41, "count": 44, "max_ms": 37.228, "p95_ms": 35.856}, "gstreamer_push": {"avg_ms": 1.035, "count": 44, "max_ms": 1.165, "p95_ms": 1.142}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.076, "count": 44, "max_ms": 38.496, "p95_ms": 37.049}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.56, "count": 44, "max_ms": 88.859, "p95_ms": 77.531}, "yolo11_640_fire_11class": {"avg_ms": 47.09, "count": 44, "max_ms": 68.354, "p95_ms": 55.467}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.185, "count": 44, "max_ms": 86.43, "p95_ms": 75.153}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.798, "count": 44, "max_ms": 184.418, "p95_ms": 172.348}, "read_interval": {"avg_ms": 597.586, "count": 43, "max_ms": 18868.284, "p95_ms": 189.811}, "triton_wait": {"avg_ms": 61.043, "count": 44, "max_ms": 88.872, "p95_ms": 77.543}, "visualize": {"avg_ms": 4.992, "count": 44, "max_ms": 5.565, "p95_ms": 5.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11924}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10861.973, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b2b6fe6540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b2b6fe6540] error while decoding MB 0 22, bytestream 3435 [h264 @ 0x55b2b7ac1a40] reference picture missing during reorder [h264 @ 0x55b2b7ac1a40] Missing reference picture, default is 65575 [h264 @ 0x55b2b7b908c0] mmco: unref short failure [h264 @ 0x55b2b7b908c0] illegal short term buffer state detected [h264 @ 0x55b2b79f2bc0] left block unavailable for requested intra mode [h264 @ 0x55b2b79f2bc0] error wh", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 476, "h264_decode_error_count": 1412, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:55:29+09:00", "last_error_summary": "[h264 @ 0x55b2b6fe6540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b2b6fe6540] error while decoding MB 0 22, bytestream 3435 [h264 @ 0x55b2b7ac1a40] reference picture missing during reorder [h264 @ 0x55b2b7ac1a40] Missing reference picture, default is 65575 [h264 @ 0x55b2b7b908c0] mmco: unref short failure [h264 @ 0x55b2b7b908c0] illegal short term buffer state detected [h264 @ 0x55b2b79f2bc0] left block unavailable for requested intra mode [h264 @ 0x55b2b79f2bc0] error wh", "last_exit_at": "2026-07-01T09:52:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 180.587, "last_started_at": "2026-07-01T09:52:28+09:00", "pid": 94806, "process": {"exists": false, "pid": 94806}, "restart_count": 523, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 524}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.323, "sample_count": 1691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1430893, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:55:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.089, "last_frame_pushed_at": "2026-07-01T09:55:29+09:00", "last_frame_read_age_sec": 0.079, "last_frame_read_at": "2026-07-01T09:55:29+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-01T09:55:29.071514+09:00", "mtime_epoch": 1782867329.071514, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157897}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:55:29+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5048.016, "name": "celery", "pid": 267, "rss_mb": 5048.016, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 73, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.441, "pushed": 2.441, "read": 2.441}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 34.049, "count": 73, "max_ms": 44.664, "p95_ms": 39.758}, "gstreamer_push": {"avg_ms": 1.077, "count": 73, "max_ms": 1.383, "p95_ms": 1.228}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.704, "count": 73, "max_ms": 39.477, "p95_ms": 35.173}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.85, "count": 73, "max_ms": 86.349, "p95_ms": 83.143}, "yolo11_640_fire_11class": {"avg_ms": 49.2, "count": 73, "max_ms": 64.285, "p95_ms": 64.104}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.442, "count": 73, "max_ms": 81.248, "p95_ms": 79.677}}, "truncated_count": 0}, "processing_total": {"avg_ms": 144.713, "count": 73, "max_ms": 186.698, "p95_ms": 176.62}, "read_interval": {"avg_ms": 410.774, "count": 73, "max_ms": 18730.688, "p95_ms": 188.936}, "triton_wait": {"avg_ms": 63.921, "count": 73, "max_ms": 86.362, "p95_ms": 83.156}, "visualize": {"avg_ms": 4.639, "count": 73, "max_ms": 6.279, "p95_ms": 5.862}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11924}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10861.609, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:55:29+09:00", "last_error_summary": "[h264 @ 0x55b2b6fe6540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b2b6fe6540] error while decoding MB 0 22, bytestream 3435 [h264 @ 0x55b2b7ac1a40] reference picture missing during reorder [h264 @ 0x55b2b7ac1a40] Missing reference picture, default is 65575 [h264 @ 0x55b2b7b908c0] mmco: unref short failure [h264 @ 0x55b2b7b908c0] illegal short term buffer state detected [h264 @ 0x55b2b79f2bc0] left block unavailable for requested intra mode [h264 @ 0x55b2b79f2bc0] error wh", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 476, "h264_decode_error_count": 1412, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:55:29+09:00", "last_error_summary": "[h264 @ 0x55b2b6fe6540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b2b6fe6540] error while decoding MB 0 22, bytestream 3435 [h264 @ 0x55b2b7ac1a40] reference picture missing during reorder [h264 @ 0x55b2b7ac1a40] Missing reference picture, default is 65575 [h264 @ 0x55b2b7b908c0] mmco: unref short failure [h264 @ 0x55b2b7b908c0] illegal short term buffer state detected [h264 @ 0x55b2b79f2bc0] left block unavailable for requested intra mode [h264 @ 0x55b2b79f2bc0] error wh", "last_exit_at": "2026-07-01T09:55:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 180.672, "last_started_at": "2026-07-01T09:52:28+09:00", "pid": 94806, "process": {"exists": false, "pid": 94806}, "restart_count": 523, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 524}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.323, "sample_count": 1691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1430894, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:55:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T09:55:29+09:00", "last_frame_read_age_sec": 0.164, "last_frame_read_at": "2026-07-01T09:55:29+09:00", "mjpeg": {"age_sec": 0.154, "exists": true, "mtime": "2026-07-01T09:55:29.071514+09:00", "mtime_epoch": 1782867329.071514, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157897}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:55:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5048.016, "name": "celery", "pid": 267, "rss_mb": 5048.016, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.468, "pushed": 2.468, "read": 2.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 34.052, "count": 73, "max_ms": 44.664, "p95_ms": 39.758}, "gstreamer_push": {"avg_ms": 1.077, "count": 74, "max_ms": 1.383, "p95_ms": 1.228}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.691, "count": 73, "max_ms": 39.477, "p95_ms": 35.173}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.657, "count": 73, "max_ms": 86.349, "p95_ms": 83.143}, "yolo11_640_fire_11class": {"avg_ms": 49.052, "count": 73, "max_ms": 64.285, "p95_ms": 64.104}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.207, "count": 73, "max_ms": 81.248, "p95_ms": 79.677}}, "truncated_count": 0}, "processing_total": {"avg_ms": 144.905, "count": 74, "max_ms": 186.698, "p95_ms": 176.62}, "read_interval": {"avg_ms": 410.774, "count": 73, "max_ms": 18730.688, "p95_ms": 188.936}, "triton_wait": {"avg_ms": 63.717, "count": 73, "max_ms": 86.362, "p95_ms": 83.156}, "visualize": {"avg_ms": 4.67, "count": 73, "max_ms": 7.411, "p95_ms": 6.191}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11924}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10861.602, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 476, "h264_decode_error_count": 1412, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:55:29+09:00", "last_error_summary": "[h264 @ 0x55b2b6fe6540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b2b6fe6540] error while decoding MB 0 22, bytestream 3435 [h264 @ 0x55b2b7ac1a40] reference picture missing during reorder [h264 @ 0x55b2b7ac1a40] Missing reference picture, default is 65575 [h264 @ 0x55b2b7b908c0] mmco: unref short failure [h264 @ 0x55b2b7b908c0] illegal short term buffer state detected [h264 @ 0x55b2b79f2bc0] left block unavailable for requested intra mode [h264 @ 0x55b2b79f2bc0] error wh", "last_exit_at": "2026-07-01T09:55:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 180.676, "last_started_at": "2026-07-01T09:52:28+09:00", "pid": 94806, "process": {"exists": false, "pid": 94806}, "restart_count": 523, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 524}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.323, "sample_count": 1691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1430894, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:55:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T09:55:29+09:00", "last_frame_read_age_sec": 0.168, "last_frame_read_at": "2026-07-01T09:55:29+09:00", "mjpeg": {"age_sec": 0.158, "exists": true, "mtime": "2026-07-01T09:55:29.071514+09:00", "mtime_epoch": 1782867329.071514, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157897}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:55:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5048.016, "name": "celery", "pid": 267, "rss_mb": 5048.016, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.468, "pushed": 2.468, "read": 2.434}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 34.052, "count": 73, "max_ms": 44.664, "p95_ms": 39.758}, "gstreamer_push": {"avg_ms": 1.077, "count": 74, "max_ms": 1.383, "p95_ms": 1.228}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.691, "count": 73, "max_ms": 39.477, "p95_ms": 35.173}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.657, "count": 73, "max_ms": 86.349, "p95_ms": 83.143}, "yolo11_640_fire_11class": {"avg_ms": 49.052, "count": 73, "max_ms": 64.285, "p95_ms": 64.104}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.207, "count": 73, "max_ms": 81.248, "p95_ms": 79.677}}, "truncated_count": 0}, "processing_total": {"avg_ms": 144.905, "count": 74, "max_ms": 186.698, "p95_ms": 176.62}, "read_interval": {"avg_ms": 410.774, "count": 73, "max_ms": 18730.688, "p95_ms": 188.936}, "triton_wait": {"avg_ms": 63.717, "count": 73, "max_ms": 86.362, "p95_ms": 83.156}, "visualize": {"avg_ms": 4.67, "count": 73, "max_ms": 7.411, "p95_ms": 6.191}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11924}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10967.043, "limit_mb": 8796093022207.996}, "detail": {"pid": 94913, "restart_count": 524, "start_count": 525}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 476, "h264_decode_error_count": 1412, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:55:29+09:00", "last_error_summary": "[h264 @ 0x55b2b6fe6540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b2b6fe6540] error while decoding MB 0 22, bytestream 3435 [h264 @ 0x55b2b7ac1a40] reference picture missing during reorder [h264 @ 0x55b2b7ac1a40] Missing reference picture, default is 65575 [h264 @ 0x55b2b7b908c0] mmco: unref short failure [h264 @ 0x55b2b7b908c0] illegal short term buffer state detected [h264 @ 0x55b2b79f2bc0] left block unavailable for requested intra mode [h264 @ 0x55b2b79f2bc0] error wh", "last_exit_at": "2026-07-01T09:55:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:55:31+09:00", "pid": 94913, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.691, "name": "ffmpeg", "pid": 94913, "rss_mb": 136.691, "state": "S (sleeping)", "threads": 106}, "restart_count": 524, "restart_count_per_hour": 25, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 525}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.116, "sample_count": 1663, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1430894, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:55:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.746, "last_frame_pushed_at": "2026-07-01T09:55:29+09:00", "last_frame_read_age_sec": 4.907, "last_frame_read_at": "2026-07-01T09:55:29+09:00", "mjpeg": {"age_sec": 4.729, "exists": true, "mtime": "2026-07-01T09:55:29.239516+09:00", "mtime_epoch": 1782867329.2395158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158229}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:55:33+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5048.016, "name": "celery", "pid": 267, "rss_mb": 5048.016, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 45, "pushed_frames": 45, "read_frames": 44, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.505, "pushed": 1.505, "read": 1.472}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.842, "count": 45, "max_ms": 41.967, "p95_ms": 39.758}, "gstreamer_push": {"avg_ms": 1.111, "count": 45, "max_ms": 1.383, "p95_ms": 1.228}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.756, "count": 45, "max_ms": 39.477, "p95_ms": 33.608}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.55, "count": 45, "max_ms": 83.591, "p95_ms": 82.638}, "yolo11_640_fire_11class": {"avg_ms": 49.677, "count": 45, "max_ms": 64.249, "p95_ms": 64.104}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.215, "count": 45, "max_ms": 81.248, "p95_ms": 80.334}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.188, "count": 45, "max_ms": 159.498, "p95_ms": 158.895}, "read_interval": {"avg_ms": 570.696, "count": 44, "max_ms": 18730.688, "p95_ms": 171.17}, "triton_wait": {"avg_ms": 63.639, "count": 45, "max_ms": 83.603, "p95_ms": 82.65}, "visualize": {"avg_ms": 4.376, "count": 45, "max_ms": 7.411, "p95_ms": 4.849}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13532}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10799.73, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5580d74366c0] error while decoding MB 66 32, bytestream -18 [in#0/rtsp @ 0x5580d7279e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580d7279e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 209, "h264_decode_error_count": 215, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:56:29+09:00", "last_error_summary": "[h264 @ 0x5580d74366c0] error while decoding MB 66 32, bytestream -18 [in#0/rtsp @ 0x5580d7279e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580d7279e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:50:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 329.829, "last_started_at": "2026-07-01T09:50:59+09:00", "pid": 94654, "process": {"exists": false, "pid": 94654}, "restart_count": 208, "restart_count_per_hour": 5, "rtsp_timeout_count": 344, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 209}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.591, "sample_count": 2582, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1623912, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:56:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T09:56:29+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-01T09:56:29+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T09:56:29.376259+09:00", "mtime_epoch": 1782867389.3762593, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230543}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:56:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1826.352, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.696, "pushed": 0.696, "read": 0.754}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.28, "count": 12, "max_ms": 21.81, "p95_ms": 21.81}, "gstreamer_push": {"avg_ms": 1.205, "count": 12, "max_ms": 1.276, "p95_ms": 1.276}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.299, "count": 12, "max_ms": 50.173, "p95_ms": 50.173}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.033, "count": 12, "max_ms": 53.119, "p95_ms": 53.119}, "yolo11_640_fire_11class": {"avg_ms": 46.929, "count": 12, "max_ms": 50.183, "p95_ms": 50.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.151, "count": 12, "max_ms": 50.658, "p95_ms": 50.658}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.48, "count": 12, "max_ms": 82.969, "p95_ms": 82.969}, "read_interval": {"avg_ms": 2367.719, "count": 13, "max_ms": 16171.289, "p95_ms": 16171.289}, "triton_wait": {"avg_ms": 51.175, "count": 12, "max_ms": 53.131, "p95_ms": 53.131}, "visualize": {"avg_ms": 3.421, "count": 12, "max_ms": 3.572, "p95_ms": 3.572}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.242, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13532}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10799.535, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:56:29+09:00", "last_error_summary": "[h264 @ 0x5580d74366c0] error while decoding MB 66 32, bytestream -18 [in#0/rtsp @ 0x5580d7279e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580d7279e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 209, "h264_decode_error_count": 215, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:56:29+09:00", "last_error_summary": "[h264 @ 0x5580d74366c0] error while decoding MB 66 32, bytestream -18 [in#0/rtsp @ 0x5580d7279e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580d7279e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:56:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 329.853, "last_started_at": "2026-07-01T09:50:59+09:00", "pid": 94654, "process": {"exists": false, "pid": 94654}, "restart_count": 208, "restart_count_per_hour": 5, "rtsp_timeout_count": 344, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 209}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.591, "sample_count": 2583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1623913, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:56:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T09:56:29+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T09:56:29+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T09:56:29.376259+09:00", "mtime_epoch": 1782867389.3762593, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230543}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:56:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1826.352, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.753, "pushed": 0.753, "read": 0.753}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.506, "count": 13, "max_ms": 22.224, "p95_ms": 22.224}, "gstreamer_push": {"avg_ms": 1.215, "count": 13, "max_ms": 1.328, "p95_ms": 1.328}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.182, "count": 13, "max_ms": 50.173, "p95_ms": 50.173}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.04, "count": 13, "max_ms": 53.119, "p95_ms": 53.119}, "yolo11_640_fire_11class": {"avg_ms": 46.86, "count": 13, "max_ms": 50.183, "p95_ms": 50.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.113, "count": 13, "max_ms": 50.658, "p95_ms": 50.658}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.823, "count": 13, "max_ms": 82.969, "p95_ms": 82.969}, "read_interval": {"avg_ms": 2367.719, "count": 13, "max_ms": 16171.289, "p95_ms": 16171.289}, "triton_wait": {"avg_ms": 51.172, "count": 13, "max_ms": 53.131, "p95_ms": 53.131}, "visualize": {"avg_ms": 3.491, "count": 13, "max_ms": 4.338, "p95_ms": 4.338}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.267, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13532}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10799.605, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 209, "h264_decode_error_count": 215, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:56:29+09:00", "last_error_summary": "[h264 @ 0x5580d74366c0] error while decoding MB 66 32, bytestream -18 [in#0/rtsp @ 0x5580d7279e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580d7279e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:56:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 329.86, "last_started_at": "2026-07-01T09:50:59+09:00", "pid": 94654, "process": {"exists": false, "pid": 94654}, "restart_count": 208, "restart_count_per_hour": 5, "rtsp_timeout_count": 344, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 209}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.591, "sample_count": 2583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1623913, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:56:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T09:56:29+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T09:56:29+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-01T09:56:29.376259+09:00", "mtime_epoch": 1782867389.3762593, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230543}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:56:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1826.352, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.753, "pushed": 0.753, "read": 0.753}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.506, "count": 13, "max_ms": 22.224, "p95_ms": 22.224}, "gstreamer_push": {"avg_ms": 1.215, "count": 13, "max_ms": 1.328, "p95_ms": 1.328}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.182, "count": 13, "max_ms": 50.173, "p95_ms": 50.173}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.04, "count": 13, "max_ms": 53.119, "p95_ms": 53.119}, "yolo11_640_fire_11class": {"avg_ms": 46.86, "count": 13, "max_ms": 50.183, "p95_ms": 50.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.113, "count": 13, "max_ms": 50.658, "p95_ms": 50.658}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.823, "count": 13, "max_ms": 82.969, "p95_ms": 82.969}, "read_interval": {"avg_ms": 2367.719, "count": 13, "max_ms": 16171.289, "p95_ms": 16171.289}, "triton_wait": {"avg_ms": 51.172, "count": 13, "max_ms": 53.131, "p95_ms": 53.131}, "visualize": {"avg_ms": 3.491, "count": 13, "max_ms": 4.338, "p95_ms": 4.338}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.273, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 69, "tracking_summary": 13532}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10819.984, "limit_mb": 8796093022207.996}, "detail": {"pid": 95020, "restart_count": 209, "start_count": 210}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 209, "h264_decode_error_count": 215, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:56:29+09:00", "last_error_summary": "[h264 @ 0x5580d74366c0] error while decoding MB 66 32, bytestream -18 [in#0/rtsp @ 0x5580d7279e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580d7279e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:56:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:56:31+09:00", "pid": 95020, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.117, "name": "ffmpeg", "pid": 95020, "rss_mb": 58.117, "state": "S (sleeping)", "threads": 1}, "restart_count": 209, "restart_count_per_hour": 6, "rtsp_timeout_count": 344, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 210}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.027, "sample_count": 2538, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1623913, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:56:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.507, "last_frame_pushed_at": "2026-07-01T09:56:29+09:00", "last_frame_read_age_sec": 4.592, "last_frame_read_at": "2026-07-01T09:56:29+09:00", "mjpeg": {"age_sec": 4.491, "exists": true, "mtime": "2026-07-01T09:56:29.472260+09:00", "mtime_epoch": 1782867389.4722605, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230372}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T09:56:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1826.352, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.597, "pushed": 0.597, "read": 0.597}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.506, "count": 13, "max_ms": 22.224, "p95_ms": 22.224}, "gstreamer_push": {"avg_ms": 1.215, "count": 13, "max_ms": 1.328, "p95_ms": 1.328}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.182, "count": 13, "max_ms": 50.173, "p95_ms": 50.173}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.04, "count": 13, "max_ms": 53.119, "p95_ms": 53.119}, "yolo11_640_fire_11class": {"avg_ms": 46.86, "count": 13, "max_ms": 50.183, "p95_ms": 50.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.113, "count": 13, "max_ms": 50.658, "p95_ms": 50.658}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.823, "count": 13, "max_ms": 82.969, "p95_ms": 82.969}, "read_interval": {"avg_ms": 2367.719, "count": 13, "max_ms": 16171.289, "p95_ms": 16171.289}, "triton_wait": {"avg_ms": 51.172, "count": 13, "max_ms": 53.131, "p95_ms": 53.131}, "visualize": {"avg_ms": 3.491, "count": 13, "max_ms": 4.338, "p95_ms": 4.338}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.765, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11932}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10862.562, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f3859e6bc0] error while decoding MB 17 9, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 477, "h264_decode_error_count": 1413, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:58:04+09:00", "last_error_summary": "[h264 @ 0x55f3859e6bc0] error while decoding MB 17 9, bytestream -5", "last_exit_at": "2026-07-01T09:55:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.415, "last_started_at": "2026-07-01T09:55:31+09:00", "pid": 94913, "process": {"exists": false, "pid": 94913}, "restart_count": 524, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 525}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.566, "sample_count": 1793, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1431860, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:58:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T09:58:04+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T09:58:04+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T09:58:04.337477+09:00", "mtime_epoch": 1782867484.3374767, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160709}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:58:04+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5048.531, "name": "celery", "pid": 267, "rss_mb": 5048.531, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.67, "pushed": 6.67, "read": 6.67}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.294, "count": 199, "max_ms": 38.212, "p95_ms": 34.937}, "gstreamer_push": {"avg_ms": 1.07, "count": 200, "max_ms": 1.361, "p95_ms": 1.155}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.912, "count": 199, "max_ms": 40.927, "p95_ms": 36.079}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.607, "count": 199, "max_ms": 88.682, "p95_ms": 83.394}, "yolo11_640_fire_11class": {"avg_ms": 48.95, "count": 199, "max_ms": 67.183, "p95_ms": 63.686}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.939, "count": 199, "max_ms": 84.55, "p95_ms": 81.274}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.734, "count": 200, "max_ms": 177.965, "p95_ms": 170.239}, "read_interval": {"avg_ms": 150.343, "count": 200, "max_ms": 190.159, "p95_ms": 182.386}, "triton_wait": {"avg_ms": 62.129, "count": 199, "max_ms": 88.694, "p95_ms": 83.406}, "visualize": {"avg_ms": 4.499, "count": 199, "max_ms": 6.095, "p95_ms": 5.476}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11932}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10862.355, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T09:58:04+09:00", "last_error_summary": "[h264 @ 0x55f3859e6bc0] error while decoding MB 17 9, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 477, "h264_decode_error_count": 1413, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:58:04+09:00", "last_error_summary": "[h264 @ 0x55f3859e6bc0] error while decoding MB 17 9, bytestream -5", "last_exit_at": "2026-07-01T09:58:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.531, "last_started_at": "2026-07-01T09:55:31+09:00", "pid": 94913, "process": {"exists": false, "pid": 94913}, "restart_count": 524, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 525}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.566, "sample_count": 1793, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1431861, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:58:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T09:58:04+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T09:58:04+09:00", "mjpeg": {"age_sec": 0.161, "exists": true, "mtime": "2026-07-01T09:58:04.337477+09:00", "mtime_epoch": 1782867484.3374767, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160709}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:58:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5048.531, "name": "celery", "pid": 267, "rss_mb": 5048.531, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.675, "pushed": 6.675, "read": 6.642}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.313, "count": 200, "max_ms": 38.212, "p95_ms": 34.937}, "gstreamer_push": {"avg_ms": 1.069, "count": 200, "max_ms": 1.361, "p95_ms": 1.155}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.909, "count": 200, "max_ms": 40.927, "p95_ms": 35.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.591, "count": 200, "max_ms": 88.682, "p95_ms": 83.368}, "yolo11_640_fire_11class": {"avg_ms": 48.918, "count": 200, "max_ms": 67.183, "p95_ms": 63.652}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.942, "count": 200, "max_ms": 84.55, "p95_ms": 80.523}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.858, "count": 200, "max_ms": 177.965, "p95_ms": 170.239}, "read_interval": {"avg_ms": 150.355, "count": 199, "max_ms": 190.159, "p95_ms": 182.606}, "triton_wait": {"avg_ms": 62.116, "count": 200, "max_ms": 88.694, "p95_ms": 83.381}, "visualize": {"avg_ms": 4.504, "count": 200, "max_ms": 6.095, "p95_ms": 5.445}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11932}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10862.348, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 477, "h264_decode_error_count": 1413, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:58:04+09:00", "last_error_summary": "[h264 @ 0x55f3859e6bc0] error while decoding MB 17 9, bytestream -5", "last_exit_at": "2026-07-01T09:58:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.535, "last_started_at": "2026-07-01T09:55:31+09:00", "pid": 94913, "process": {"exists": false, "pid": 94913}, "restart_count": 524, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 525}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.566, "sample_count": 1793, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1431861, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:58:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T09:58:04+09:00", "last_frame_read_age_sec": 0.171, "last_frame_read_at": "2026-07-01T09:58:04+09:00", "mjpeg": {"age_sec": 0.164, "exists": true, "mtime": "2026-07-01T09:58:04.337477+09:00", "mtime_epoch": 1782867484.3374767, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160709}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:58:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5048.531, "name": "celery", "pid": 267, "rss_mb": 5048.531, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.674, "pushed": 6.674, "read": 6.641}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.313, "count": 200, "max_ms": 38.212, "p95_ms": 34.937}, "gstreamer_push": {"avg_ms": 1.069, "count": 200, "max_ms": 1.361, "p95_ms": 1.155}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.909, "count": 200, "max_ms": 40.927, "p95_ms": 35.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.591, "count": 200, "max_ms": 88.682, "p95_ms": 83.368}, "yolo11_640_fire_11class": {"avg_ms": 48.918, "count": 200, "max_ms": 67.183, "p95_ms": 63.652}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.942, "count": 200, "max_ms": 84.55, "p95_ms": 80.523}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.858, "count": 200, "max_ms": 177.965, "p95_ms": 170.239}, "read_interval": {"avg_ms": 150.355, "count": 199, "max_ms": 190.159, "p95_ms": 182.606}, "triton_wait": {"avg_ms": 62.116, "count": 200, "max_ms": 88.694, "p95_ms": 83.381}, "visualize": {"avg_ms": 4.504, "count": 200, "max_ms": 6.095, "p95_ms": 5.445}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11932}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10967.602, "limit_mb": 8796093022207.996}, "detail": {"pid": 95172, "restart_count": 525, "start_count": 526}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 477, "h264_decode_error_count": 1413, "h264_decode_error_count_per_hour": 36, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T09:58:04+09:00", "last_error_summary": "[h264 @ 0x55f3859e6bc0] error while decoding MB 17 9, bytestream -5", "last_exit_at": "2026-07-01T09:58:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T09:58:07+09:00", "pid": 95172, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.84, "name": "ffmpeg", "pid": 95172, "rss_mb": 136.84, "state": "S (sleeping)", "threads": 106}, "restart_count": 525, "restart_count_per_hour": 25, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 526}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.566, "sample_count": 1761, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1431861, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T09:58:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.898, "last_frame_pushed_at": "2026-07-01T09:58:04+09:00", "last_frame_read_age_sec": 5.06, "last_frame_read_at": "2026-07-01T09:58:04+09:00", "mjpeg": {"age_sec": 4.882, "exists": true, "mtime": "2026-07-01T09:58:04.509479+09:00", "mtime_epoch": 1782867484.509479, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155044}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T09:58:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5048.531, "name": "celery", "pid": 267, "rss_mb": 5048.531, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 167, "pushed_frames": 167, "read_frames": 166, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.589, "pushed": 5.589, "read": 5.555}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.175, "count": 167, "max_ms": 38.212, "p95_ms": 34.876}, "gstreamer_push": {"avg_ms": 1.067, "count": 167, "max_ms": 1.361, "p95_ms": 1.159}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.956, "count": 167, "max_ms": 40.927, "p95_ms": 35.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.186, "count": 167, "max_ms": 86.392, "p95_ms": 83.368}, "yolo11_640_fire_11class": {"avg_ms": 48.914, "count": 167, "max_ms": 67.104, "p95_ms": 63.652}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.03, "count": 167, "max_ms": 84.55, "p95_ms": 81.274}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.824, "count": 167, "max_ms": 177.965, "p95_ms": 171.806}, "read_interval": {"avg_ms": 150.284, "count": 166, "max_ms": 190.159, "p95_ms": 182.606}, "triton_wait": {"avg_ms": 61.813, "count": 167, "max_ms": 86.404, "p95_ms": 83.381}, "visualize": {"avg_ms": 4.524, "count": 167, "max_ms": 6.095, "p95_ms": 5.487}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10864.648, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56544fedc700] error while decoding MB 65 3, bytestream -32", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 478, "h264_decode_error_count": 1414, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:00:35+09:00", "last_error_summary": "[h264 @ 0x56544fedc700] error while decoding MB 65 3, bytestream -32", "last_exit_at": "2026-07-01T09:58:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.351, "last_started_at": "2026-07-01T09:58:07+09:00", "pid": 95172, "process": {"exists": false, "pid": 95172}, "restart_count": 525, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 526}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.463, "sample_count": 1873, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1432777, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:00:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-01T10:00:35+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-01T10:00:35+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T10:00:35.695507+09:00", "mtime_epoch": 1782867635.6955073, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165114}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:00:35+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5048.789, "name": "celery", "pid": 267, "rss_mb": 5048.789, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.269, "pushed": 6.269, "read": 6.302}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.415, "count": 188, "max_ms": 42.032, "p95_ms": 35.362}, "gstreamer_push": {"avg_ms": 1.015, "count": 188, "max_ms": 1.202, "p95_ms": 1.124}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.042, "count": 188, "max_ms": 40.993, "p95_ms": 35.798}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.008, "count": 188, "max_ms": 85.641, "p95_ms": 82.021}, "yolo11_640_fire_11class": {"avg_ms": 48.108, "count": 188, "max_ms": 67.659, "p95_ms": 62.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.301, "count": 188, "max_ms": 81.998, "p95_ms": 78.825}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.985, "count": 188, "max_ms": 182.48, "p95_ms": 174.665}, "read_interval": {"avg_ms": 159.092, "count": 189, "max_ms": 193.106, "p95_ms": 185.36}, "triton_wait": {"avg_ms": 61.222, "count": 188, "max_ms": 85.654, "p95_ms": 82.033}, "visualize": {"avg_ms": 4.894, "count": 188, "max_ms": 6.591, "p95_ms": 5.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10863.773, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:00:35+09:00", "last_error_summary": "[h264 @ 0x56544fedc700] error while decoding MB 65 3, bytestream -32", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 478, "h264_decode_error_count": 1414, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:00:35+09:00", "last_error_summary": "[h264 @ 0x56544fedc700] error while decoding MB 65 3, bytestream -32", "last_exit_at": "2026-07-01T10:00:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.465, "last_started_at": "2026-07-01T09:58:07+09:00", "pid": 95172, "process": {"exists": false, "pid": 95172}, "restart_count": 525, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 526}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.463, "sample_count": 1874, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1432778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:00:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T10:00:35+09:00", "last_frame_read_age_sec": 0.17, "last_frame_read_at": "2026-07-01T10:00:35+09:00", "mjpeg": {"age_sec": 0.16, "exists": true, "mtime": "2026-07-01T10:00:35.695507+09:00", "mtime_epoch": 1782867635.6955073, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165114}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:00:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5048.789, "name": "celery", "pid": 267, "rss_mb": 5048.789, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.303, "pushed": 6.303, "read": 6.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.423, "count": 188, "max_ms": 42.032, "p95_ms": 35.362}, "gstreamer_push": {"avg_ms": 1.015, "count": 189, "max_ms": 1.202, "p95_ms": 1.124}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.032, "count": 188, "max_ms": 40.993, "p95_ms": 35.798}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.085, "count": 188, "max_ms": 85.641, "p95_ms": 82.021}, "yolo11_640_fire_11class": {"avg_ms": 48.137, "count": 188, "max_ms": 67.659, "p95_ms": 62.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.383, "count": 188, "max_ms": 81.998, "p95_ms": 78.825}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.073, "count": 189, "max_ms": 182.48, "p95_ms": 174.665}, "read_interval": {"avg_ms": 159.229, "count": 188, "max_ms": 193.106, "p95_ms": 185.36}, "triton_wait": {"avg_ms": 61.299, "count": 188, "max_ms": 85.654, "p95_ms": 82.033}, "visualize": {"avg_ms": 4.9, "count": 188, "max_ms": 6.591, "p95_ms": 5.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10863.766, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 478, "h264_decode_error_count": 1414, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:00:35+09:00", "last_error_summary": "[h264 @ 0x56544fedc700] error while decoding MB 65 3, bytestream -32", "last_exit_at": "2026-07-01T10:00:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.469, "last_started_at": "2026-07-01T09:58:07+09:00", "pid": 95172, "process": {"exists": false, "pid": 95172}, "restart_count": 525, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 526}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.463, "sample_count": 1874, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1432778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:00:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T10:00:35+09:00", "last_frame_read_age_sec": 0.174, "last_frame_read_at": "2026-07-01T10:00:35+09:00", "mjpeg": {"age_sec": 0.164, "exists": true, "mtime": "2026-07-01T10:00:35.695507+09:00", "mtime_epoch": 1782867635.6955073, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165114}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:00:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5048.789, "name": "celery", "pid": 267, "rss_mb": 5048.789, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.303, "pushed": 6.303, "read": 6.269}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.423, "count": 188, "max_ms": 42.032, "p95_ms": 35.362}, "gstreamer_push": {"avg_ms": 1.015, "count": 189, "max_ms": 1.202, "p95_ms": 1.124}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.032, "count": 188, "max_ms": 40.993, "p95_ms": 35.798}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.085, "count": 188, "max_ms": 85.641, "p95_ms": 82.021}, "yolo11_640_fire_11class": {"avg_ms": 48.137, "count": 188, "max_ms": 67.659, "p95_ms": 62.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.383, "count": 188, "max_ms": 81.998, "p95_ms": 78.825}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.073, "count": 189, "max_ms": 182.48, "p95_ms": 174.665}, "read_interval": {"avg_ms": 159.229, "count": 188, "max_ms": 193.106, "p95_ms": 185.36}, "triton_wait": {"avg_ms": 61.299, "count": 188, "max_ms": 85.654, "p95_ms": 82.033}, "visualize": {"avg_ms": 4.9, "count": 188, "max_ms": 6.591, "p95_ms": 5.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10972.906, "limit_mb": 8796093022207.996}, "detail": {"pid": 95279, "restart_count": 526, "start_count": 527}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 478, "h264_decode_error_count": 1414, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:00:35+09:00", "last_error_summary": "[h264 @ 0x56544fedc700] error while decoding MB 65 3, bytestream -32", "last_exit_at": "2026-07-01T10:00:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:00:38+09:00", "pid": 95279, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.352, "name": "ffmpeg", "pid": 95279, "rss_mb": 139.352, "state": "S (sleeping)", "threads": 106}, "restart_count": 526, "restart_count_per_hour": 25, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 527}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.501, "sample_count": 1842, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1432778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:00:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.937, "last_frame_pushed_at": "2026-07-01T10:00:35+09:00", "last_frame_read_age_sec": 5.104, "last_frame_read_at": "2026-07-01T10:00:35+09:00", "mjpeg": {"age_sec": 4.921, "exists": true, "mtime": "2026-07-01T10:00:35.867510+09:00", "mtime_epoch": 1782867635.8675096, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165111}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:00:40+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5048.789, "name": "celery", "pid": 267, "rss_mb": 5048.789, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 156, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.201, "pushed": 5.201, "read": 5.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.521, "count": 156, "max_ms": 42.032, "p95_ms": 35.34}, "gstreamer_push": {"avg_ms": 1.02, "count": 156, "max_ms": 1.202, "p95_ms": 1.127}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.226, "count": 156, "max_ms": 40.993, "p95_ms": 36.032}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.81, "count": 156, "max_ms": 85.641, "p95_ms": 82.079}, "yolo11_640_fire_11class": {"avg_ms": 48.576, "count": 156, "max_ms": 67.659, "p95_ms": 62.757}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.023, "count": 156, "max_ms": 81.998, "p95_ms": 79.159}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.421, "count": 156, "max_ms": 182.48, "p95_ms": 175.366}, "read_interval": {"avg_ms": 160.556, "count": 156, "max_ms": 193.106, "p95_ms": 185.421}, "triton_wait": {"avg_ms": 62.059, "count": 156, "max_ms": 85.654, "p95_ms": 82.092}, "visualize": {"avg_ms": 4.907, "count": 156, "max_ms": 6.591, "p95_ms": 5.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11132.309, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a8a0e57780] error while decoding MB 15 11, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 479, "h264_decode_error_count": 1415, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:03:03+09:00", "last_error_summary": "[h264 @ 0x55a8a0e57780] error while decoding MB 15 11, bytestream -5", "last_exit_at": "2026-07-01T10:00:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 144.773, "last_started_at": "2026-07-01T10:00:38+09:00", "pid": 95279, "process": {"exists": false, "pid": 95279}, "restart_count": 526, "restart_count_per_hour": 25, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 527}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.684, "sample_count": 1820, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1433674, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:03:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T10:03:03+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T10:03:03+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T10:03:03.517574+09:00", "mtime_epoch": 1782867783.5175736, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159247}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:03:03+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5049.305, "name": "celery", "pid": 267, "rss_mb": 5049.305, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.746, "pushed": 6.746, "read": 6.746}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.001, "count": 202, "max_ms": 45.462, "p95_ms": 37.938}, "gstreamer_push": {"avg_ms": 1.073, "count": 202, "max_ms": 1.421, "p95_ms": 1.157}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.263, "count": 202, "max_ms": 53.3, "p95_ms": 38.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.38, "count": 202, "max_ms": 91.771, "p95_ms": 84.862}, "yolo11_640_fire_11class": {"avg_ms": 48.038, "count": 202, "max_ms": 65.953, "p95_ms": 62.918}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.02, "count": 202, "max_ms": 92.502, "p95_ms": 81.781}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.728, "count": 202, "max_ms": 180.329, "p95_ms": 159.27}, "read_interval": {"avg_ms": 148.629, "count": 202, "max_ms": 192.471, "p95_ms": 171.391}, "triton_wait": {"avg_ms": 62.626, "count": 202, "max_ms": 92.514, "p95_ms": 85.206}, "visualize": {"avg_ms": 4.379, "count": 202, "max_ms": 6.852, "p95_ms": 5.426}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11129.215, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:03:03+09:00", "last_error_summary": "[h264 @ 0x55a8a0e57780] error while decoding MB 15 11, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 479, "h264_decode_error_count": 1415, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:03:03+09:00", "last_error_summary": "[h264 @ 0x55a8a0e57780] error while decoding MB 15 11, bytestream -5", "last_exit_at": "2026-07-01T10:03:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 144.862, "last_started_at": "2026-07-01T10:00:38+09:00", "pid": 95279, "process": {"exists": false, "pid": 95279}, "restart_count": 526, "restart_count_per_hour": 25, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 527}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.684, "sample_count": 1820, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1433675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:03:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T10:03:03+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-01T10:03:03+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-01T10:03:03.517574+09:00", "mtime_epoch": 1782867783.5175736, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159247}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:03:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5049.305, "name": "celery", "pid": 267, "rss_mb": 5049.305, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.76, "pushed": 6.76, "read": 6.726}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.02, "count": 202, "max_ms": 45.462, "p95_ms": 37.938}, "gstreamer_push": {"avg_ms": 1.073, "count": 202, "max_ms": 1.421, "p95_ms": 1.157}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.246, "count": 202, "max_ms": 53.3, "p95_ms": 38.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.341, "count": 202, "max_ms": 91.771, "p95_ms": 84.862}, "yolo11_640_fire_11class": {"avg_ms": 48.013, "count": 202, "max_ms": 65.953, "p95_ms": 62.918}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.004, "count": 202, "max_ms": 92.502, "p95_ms": 81.781}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.739, "count": 202, "max_ms": 180.329, "p95_ms": 159.27}, "read_interval": {"avg_ms": 148.656, "count": 201, "max_ms": 192.471, "p95_ms": 171.391}, "triton_wait": {"avg_ms": 62.587, "count": 202, "max_ms": 92.514, "p95_ms": 85.206}, "visualize": {"avg_ms": 4.384, "count": 202, "max_ms": 6.852, "p95_ms": 5.426}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11129.207, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 479, "h264_decode_error_count": 1415, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:03:03+09:00", "last_error_summary": "[h264 @ 0x55a8a0e57780] error while decoding MB 15 11, bytestream -5", "last_exit_at": "2026-07-01T10:03:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 144.867, "last_started_at": "2026-07-01T10:00:38+09:00", "pid": 95279, "process": {"exists": false, "pid": 95279}, "restart_count": 526, "restart_count_per_hour": 25, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 527}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.684, "sample_count": 1820, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1433675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:03:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T10:03:03+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-01T10:03:03+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-01T10:03:03.517574+09:00", "mtime_epoch": 1782867783.5175736, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159247}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:03:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5049.305, "name": "celery", "pid": 267, "rss_mb": 5049.305, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.759, "pushed": 6.759, "read": 6.725}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.02, "count": 202, "max_ms": 45.462, "p95_ms": 37.938}, "gstreamer_push": {"avg_ms": 1.073, "count": 202, "max_ms": 1.421, "p95_ms": 1.157}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.246, "count": 202, "max_ms": 53.3, "p95_ms": 38.654}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.341, "count": 202, "max_ms": 91.771, "p95_ms": 84.862}, "yolo11_640_fire_11class": {"avg_ms": 48.013, "count": 202, "max_ms": 65.953, "p95_ms": 62.918}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.004, "count": 202, "max_ms": 92.502, "p95_ms": 81.781}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.739, "count": 202, "max_ms": 180.329, "p95_ms": 159.27}, "read_interval": {"avg_ms": 148.656, "count": 201, "max_ms": 192.471, "p95_ms": 171.391}, "triton_wait": {"avg_ms": 62.587, "count": 202, "max_ms": 92.514, "p95_ms": 85.206}, "visualize": {"avg_ms": 4.384, "count": 202, "max_ms": 6.852, "p95_ms": 5.426}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11245.051, "limit_mb": 8796093022207.996}, "detail": {"pid": 95446, "restart_count": 527, "start_count": 528}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 479, "h264_decode_error_count": 1415, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:03:03+09:00", "last_error_summary": "[h264 @ 0x55a8a0e57780] error while decoding MB 15 11, bytestream -5", "last_exit_at": "2026-07-01T10:03:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:03:06+09:00", "pid": 95446, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.176, "name": "ffmpeg", "pid": 95446, "rss_mb": 144.176, "state": "S (sleeping)", "threads": 106}, "restart_count": 527, "restart_count_per_hour": 26, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 528}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.684, "sample_count": 1814, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1433675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:03:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.669, "last_frame_pushed_at": "2026-07-01T10:03:03+09:00", "last_frame_read_age_sec": 4.804, "last_frame_read_at": "2026-07-01T10:03:03+09:00", "mjpeg": {"age_sec": 4.653, "exists": true, "mtime": "2026-07-01T10:03:03.661576+09:00", "mtime_epoch": 1782867783.6615756, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159872}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:03:08+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5049.305, "name": "celery", "pid": 267, "rss_mb": 5049.305, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 171, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.736, "pushed": 5.736, "read": 5.702}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.887, "count": 171, "max_ms": 45.462, "p95_ms": 37.938}, "gstreamer_push": {"avg_ms": 1.064, "count": 172, "max_ms": 1.223, "p95_ms": 1.157}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.224, "count": 171, "max_ms": 53.3, "p95_ms": 39.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.018, "count": 171, "max_ms": 91.771, "p95_ms": 85.194}, "yolo11_640_fire_11class": {"avg_ms": 47.742, "count": 171, "max_ms": 65.953, "p95_ms": 63.802}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.971, "count": 171, "max_ms": 92.502, "p95_ms": 82.4}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.194, "count": 172, "max_ms": 180.329, "p95_ms": 161.799}, "read_interval": {"avg_ms": 148.089, "count": 171, "max_ms": 192.471, "p95_ms": 173.628}, "triton_wait": {"avg_ms": 62.252, "count": 171, "max_ms": 92.514, "p95_ms": 85.847}, "visualize": {"avg_ms": 4.382, "count": 171, "max_ms": 6.852, "p95_ms": 5.498}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13572}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10963.367, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55fbe09857c0] error while decoding MB 57 35, bytestream -30 [in#0/rtsp @ 0x55fbe02ece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fbe02ece00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 210, "h264_decode_error_count": 216, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:05:31+09:00", "last_error_summary": "[h264 @ 0x55fbe09857c0] error while decoding MB 57 35, bytestream -30 [in#0/rtsp @ 0x55fbe02ece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fbe02ece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T09:56:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 539.894, "last_started_at": "2026-07-01T09:56:31+09:00", "pid": 95020, "process": {"exists": false, "pid": 95020}, "restart_count": 209, "restart_count_per_hour": 5, "rtsp_timeout_count": 346, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 210}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.212, "sample_count": 2438, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1628661, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:05:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T10:05:31+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-01T10:05:31+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-01T10:05:31.799708+09:00", "mtime_epoch": 1782867931.7997077, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218751}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:05:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1972.336, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.424, "pushed": 9.424, "read": 10.209}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.249, "count": 12, "max_ms": 21.957, "p95_ms": 21.957}, "gstreamer_push": {"avg_ms": 1.159, "count": 12, "max_ms": 1.261, "p95_ms": 1.261}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.922, "count": 12, "max_ms": 60.702, "p95_ms": 60.702}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.4, "count": 12, "max_ms": 80.363, "p95_ms": 80.363}, "yolo11_640_fire_11class": {"avg_ms": 50.004, "count": 12, "max_ms": 61.309, "p95_ms": 61.309}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.232, "count": 12, "max_ms": 77.63, "p95_ms": 77.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.729, "count": 12, "max_ms": 108.174, "p95_ms": 108.174}, "read_interval": {"avg_ms": 2316.813, "count": 13, "max_ms": 28907.462, "p95_ms": 28907.462}, "triton_wait": {"avg_ms": 59.412, "count": 12, "max_ms": 80.375, "p95_ms": 80.375}, "visualize": {"avg_ms": 3.38, "count": 12, "max_ms": 3.569, "p95_ms": 3.569}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.273, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13572}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10962.988, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:05:31+09:00", "last_error_summary": "[h264 @ 0x55fbe09857c0] error while decoding MB 57 35, bytestream -30 [in#0/rtsp @ 0x55fbe02ece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fbe02ece00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 210, "h264_decode_error_count": 216, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:05:31+09:00", "last_error_summary": "[h264 @ 0x55fbe09857c0] error while decoding MB 57 35, bytestream -30 [in#0/rtsp @ 0x55fbe02ece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fbe02ece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:05:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 539.945, "last_started_at": "2026-07-01T09:56:31+09:00", "pid": 95020, "process": {"exists": false, "pid": 95020}, "restart_count": 209, "restart_count_per_hour": 5, "rtsp_timeout_count": 346, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 210}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.212, "sample_count": 2439, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1628662, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:05:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T10:05:31+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T10:05:31+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T10:05:31.799708+09:00", "mtime_epoch": 1782867931.7997077, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218751}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:05:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1972.336, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.817, "pushed": 9.817, "read": 9.817}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.194, "count": 13, "max_ms": 21.957, "p95_ms": 21.957}, "gstreamer_push": {"avg_ms": 1.164, "count": 13, "max_ms": 1.261, "p95_ms": 1.261}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.907, "count": 13, "max_ms": 60.702, "p95_ms": 60.702}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.877, "count": 13, "max_ms": 80.363, "p95_ms": 80.363}, "yolo11_640_fire_11class": {"avg_ms": 50.633, "count": 13, "max_ms": 61.309, "p95_ms": 61.309}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.666, "count": 13, "max_ms": 77.63, "p95_ms": 77.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.138, "count": 13, "max_ms": 108.174, "p95_ms": 108.174}, "read_interval": {"avg_ms": 2316.813, "count": 13, "max_ms": 28907.462, "p95_ms": 28907.462}, "triton_wait": {"avg_ms": 60.889, "count": 13, "max_ms": 80.375, "p95_ms": 80.375}, "visualize": {"avg_ms": 3.377, "count": 13, "max_ms": 3.569, "p95_ms": 3.569}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.324, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13572}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10962.973, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 210, "h264_decode_error_count": 216, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:05:31+09:00", "last_error_summary": "[h264 @ 0x55fbe09857c0] error while decoding MB 57 35, bytestream -30 [in#0/rtsp @ 0x55fbe02ece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fbe02ece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:05:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 539.951, "last_started_at": "2026-07-01T09:56:31+09:00", "pid": 95020, "process": {"exists": false, "pid": 95020}, "restart_count": 209, "restart_count_per_hour": 5, "rtsp_timeout_count": 346, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 210}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.212, "sample_count": 2439, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1628662, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:05:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T10:05:31+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-01T10:05:31+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-01T10:05:31.799708+09:00", "mtime_epoch": 1782867931.7997077, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218751}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:05:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1972.336, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.773, "pushed": 9.773, "read": 9.773}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.194, "count": 13, "max_ms": 21.957, "p95_ms": 21.957}, "gstreamer_push": {"avg_ms": 1.164, "count": 13, "max_ms": 1.261, "p95_ms": 1.261}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.907, "count": 13, "max_ms": 60.702, "p95_ms": 60.702}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.877, "count": 13, "max_ms": 80.363, "p95_ms": 80.363}, "yolo11_640_fire_11class": {"avg_ms": 50.633, "count": 13, "max_ms": 61.309, "p95_ms": 61.309}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.666, "count": 13, "max_ms": 77.63, "p95_ms": 77.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.138, "count": 13, "max_ms": 108.174, "p95_ms": 108.174}, "read_interval": {"avg_ms": 2316.813, "count": 13, "max_ms": 28907.462, "p95_ms": 28907.462}, "triton_wait": {"avg_ms": 60.889, "count": 13, "max_ms": 80.375, "p95_ms": 80.375}, "visualize": {"avg_ms": 3.377, "count": 13, "max_ms": 3.569, "p95_ms": 3.569}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.33, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13572}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 11086.062, "limit_mb": 8796093022207.996}, "detail": {"pid": 95553, "restart_count": 210, "start_count": 211}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 210, "h264_decode_error_count": 216, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:05:31+09:00", "last_error_summary": "[h264 @ 0x55fbe09857c0] error while decoding MB 57 35, bytestream -30 [in#0/rtsp @ 0x55fbe02ece00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fbe02ece00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:05:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:05:34+09:00", "pid": 95553, "process": {"exists": true, "fd_count": 5, "hwm_mb": 157.219, "name": "ffmpeg", "pid": 95553, "rss_mb": 157.055, "state": "R (running)", "threads": 49}, "restart_count": 210, "restart_count_per_hour": 6, "rtsp_timeout_count": 346, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 211}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.437, "sample_count": 2394, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1628662, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:05:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.471, "last_frame_pushed_at": "2026-07-01T10:05:31+09:00", "last_frame_read_age_sec": 4.579, "last_frame_read_at": "2026-07-01T10:05:31+09:00", "mjpeg": {"age_sec": 4.453, "exists": true, "mtime": "2026-07-01T10:05:31.919709+09:00", "mtime_epoch": 1782867931.9197094, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 215690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:05:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1972.336, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.245, "pushed": 2.245, "read": 2.245}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.194, "count": 13, "max_ms": 21.957, "p95_ms": 21.957}, "gstreamer_push": {"avg_ms": 1.164, "count": 13, "max_ms": 1.261, "p95_ms": 1.261}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.907, "count": 13, "max_ms": 60.702, "p95_ms": 60.702}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.877, "count": 13, "max_ms": 80.363, "p95_ms": 80.363}, "yolo11_640_fire_11class": {"avg_ms": 50.633, "count": 13, "max_ms": 61.309, "p95_ms": 61.309}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.666, "count": 13, "max_ms": 77.63, "p95_ms": 77.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.138, "count": 13, "max_ms": 108.174, "p95_ms": 108.174}, "read_interval": {"avg_ms": 2316.813, "count": 13, "max_ms": 28907.462, "p95_ms": 28907.462}, "triton_wait": {"avg_ms": 60.889, "count": 13, "max_ms": 80.375, "p95_ms": 80.375}, "visualize": {"avg_ms": 3.377, "count": 13, "max_ms": 3.569, "p95_ms": 3.569}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.79, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11955}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11020.699, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5647dc7d2440] error while decoding MB 56 22, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 480, "h264_decode_error_count": 1416, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:05:44+09:00", "last_error_summary": "[h264 @ 0x5647dc7d2440] error while decoding MB 56 22, bytestream -5", "last_exit_at": "2026-07-01T10:03:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.476, "last_started_at": "2026-07-01T10:03:06+09:00", "pid": 95446, "process": {"exists": false, "pid": 95446}, "restart_count": 527, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 528}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.03, "sample_count": 1906, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1434707, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:05:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T10:05:44+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T10:05:44+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T10:05:44.747896+09:00", "mtime_epoch": 1782867944.7478964, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161368}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:05:44+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5049.82, "name": "celery", "pid": 267, "rss_mb": 5049.82, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.806, "pushed": 6.806, "read": 6.806}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.04, "count": 203, "max_ms": 41.012, "p95_ms": 33.536}, "gstreamer_push": {"avg_ms": 1.029, "count": 204, "max_ms": 1.339, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.934, "count": 203, "max_ms": 41.698, "p95_ms": 34.794}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.152, "count": 203, "max_ms": 84.382, "p95_ms": 71.101}, "yolo11_640_fire_11class": {"avg_ms": 45.316, "count": 203, "max_ms": 65.818, "p95_ms": 52.569}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.336, "count": 203, "max_ms": 80.905, "p95_ms": 68.134}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.859, "count": 204, "max_ms": 268.501, "p95_ms": 158.606}, "read_interval": {"avg_ms": 147.267, "count": 204, "max_ms": 278.248, "p95_ms": 169.784}, "triton_wait": {"avg_ms": 56.18, "count": 203, "max_ms": 84.394, "p95_ms": 71.113}, "visualize": {"avg_ms": 4.71, "count": 203, "max_ms": 5.94, "p95_ms": 5.465}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11955}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11020.016, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:05:44+09:00", "last_error_summary": "[h264 @ 0x5647dc7d2440] error while decoding MB 56 22, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 480, "h264_decode_error_count": 1416, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:05:44+09:00", "last_error_summary": "[h264 @ 0x5647dc7d2440] error while decoding MB 56 22, bytestream -5", "last_exit_at": "2026-07-01T10:05:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.593, "last_started_at": "2026-07-01T10:03:06+09:00", "pid": 95446, "process": {"exists": false, "pid": 95446}, "restart_count": 527, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 528}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.03, "sample_count": 1906, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1434708, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:05:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T10:05:44+09:00", "last_frame_read_age_sec": 0.169, "last_frame_read_at": "2026-07-01T10:05:44+09:00", "mjpeg": {"age_sec": 0.158, "exists": true, "mtime": "2026-07-01T10:05:44.747896+09:00", "mtime_epoch": 1782867944.7478964, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161368}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:05:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5049.82, "name": "celery", "pid": 267, "rss_mb": 5049.82, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.81, "pushed": 6.81, "read": 6.776}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.069, "count": 204, "max_ms": 41.012, "p95_ms": 33.568}, "gstreamer_push": {"avg_ms": 1.029, "count": 204, "max_ms": 1.339, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.937, "count": 204, "max_ms": 41.698, "p95_ms": 34.794}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.174, "count": 204, "max_ms": 84.382, "p95_ms": 71.101}, "yolo11_640_fire_11class": {"avg_ms": 45.33, "count": 204, "max_ms": 65.818, "p95_ms": 52.569}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.358, "count": 204, "max_ms": 80.905, "p95_ms": 68.134}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.06, "count": 204, "max_ms": 268.501, "p95_ms": 159.102}, "read_interval": {"avg_ms": 147.34, "count": 203, "max_ms": 278.248, "p95_ms": 169.784}, "triton_wait": {"avg_ms": 56.203, "count": 204, "max_ms": 84.394, "p95_ms": 71.113}, "visualize": {"avg_ms": 4.713, "count": 204, "max_ms": 5.94, "p95_ms": 5.465}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11955}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11020.008, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 480, "h264_decode_error_count": 1416, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:05:44+09:00", "last_error_summary": "[h264 @ 0x5647dc7d2440] error while decoding MB 56 22, bytestream -5", "last_exit_at": "2026-07-01T10:05:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.597, "last_started_at": "2026-07-01T10:03:06+09:00", "pid": 95446, "process": {"exists": false, "pid": 95446}, "restart_count": 527, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 528}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.03, "sample_count": 1906, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1434708, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:05:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T10:05:44+09:00", "last_frame_read_age_sec": 0.173, "last_frame_read_at": "2026-07-01T10:05:44+09:00", "mjpeg": {"age_sec": 0.162, "exists": true, "mtime": "2026-07-01T10:05:44.747896+09:00", "mtime_epoch": 1782867944.7478964, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161368}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:05:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5049.82, "name": "celery", "pid": 267, "rss_mb": 5049.82, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.809, "pushed": 6.809, "read": 6.776}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.069, "count": 204, "max_ms": 41.012, "p95_ms": 33.568}, "gstreamer_push": {"avg_ms": 1.029, "count": 204, "max_ms": 1.339, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.937, "count": 204, "max_ms": 41.698, "p95_ms": 34.794}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.174, "count": 204, "max_ms": 84.382, "p95_ms": 71.101}, "yolo11_640_fire_11class": {"avg_ms": 45.33, "count": 204, "max_ms": 65.818, "p95_ms": 52.569}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.358, "count": 204, "max_ms": 80.905, "p95_ms": 68.134}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.06, "count": 204, "max_ms": 268.501, "p95_ms": 159.102}, "read_interval": {"avg_ms": 147.34, "count": 203, "max_ms": 278.248, "p95_ms": 169.784}, "triton_wait": {"avg_ms": 56.203, "count": 204, "max_ms": 84.394, "p95_ms": 71.113}, "visualize": {"avg_ms": 4.713, "count": 204, "max_ms": 5.94, "p95_ms": 5.465}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11955}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11132.363, "limit_mb": 8796093022207.996}, "detail": {"pid": 95705, "restart_count": 528, "start_count": 529}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 480, "h264_decode_error_count": 1416, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:05:44+09:00", "last_error_summary": "[h264 @ 0x5647dc7d2440] error while decoding MB 56 22, bytestream -5", "last_exit_at": "2026-07-01T10:05:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:05:47+09:00", "pid": 95705, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.582, "name": "ffmpeg", "pid": 95705, "rss_mb": 141.582, "state": "S (sleeping)", "threads": 106}, "restart_count": 528, "restart_count_per_hour": 25, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 529}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.96, "sample_count": 1881, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1434708, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:05:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.382, "last_frame_pushed_at": "2026-07-01T10:05:44+09:00", "last_frame_read_age_sec": 4.545, "last_frame_read_at": "2026-07-01T10:05:44+09:00", "mjpeg": {"age_sec": 4.363, "exists": true, "mtime": "2026-07-01T10:05:44.919899+09:00", "mtime_epoch": 1782867944.9198987, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158174}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:05:49+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5049.82, "name": "celery", "pid": 267, "rss_mb": 5049.82, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.808, "pushed": 5.808, "read": 5.774}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.199, "count": 174, "max_ms": 41.012, "p95_ms": 33.97}, "gstreamer_push": {"avg_ms": 1.029, "count": 174, "max_ms": 1.339, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.051, "count": 174, "max_ms": 41.698, "p95_ms": 35.099}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.737, "count": 174, "max_ms": 84.382, "p95_ms": 78.734}, "yolo11_640_fire_11class": {"avg_ms": 45.713, "count": 174, "max_ms": 65.818, "p95_ms": 53.179}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.949, "count": 174, "max_ms": 80.905, "p95_ms": 74.196}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.313, "count": 174, "max_ms": 268.501, "p95_ms": 160.038}, "read_interval": {"avg_ms": 147.589, "count": 173, "max_ms": 278.248, "p95_ms": 170.056}, "triton_wait": {"avg_ms": 56.769, "count": 174, "max_ms": 84.394, "p95_ms": 78.745}, "visualize": {"avg_ms": 4.69, "count": 174, "max_ms": 5.94, "p95_ms": 5.461}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11964}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11025.539, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fb6efb26c0] error while decoding MB 77 37, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 481, "h264_decode_error_count": 1417, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:08:27+09:00", "last_error_summary": "[h264 @ 0x55fb6efb26c0] error while decoding MB 77 37, bytestream -5", "last_exit_at": "2026-07-01T10:05:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.939, "last_started_at": "2026-07-01T10:05:47+09:00", "pid": 95705, "process": {"exists": false, "pid": 95705}, "restart_count": 528, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 529}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.803, "sample_count": 1960, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1435760, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:08:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T10:08:27+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T10:08:27+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T10:08:27.178290+09:00", "mtime_epoch": 1782868107.1782901, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151478}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:08:27+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5050.336, "name": "celery", "pid": 267, "rss_mb": 5050.336, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.851, "pushed": 6.851, "read": 6.851}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.951, "count": 205, "max_ms": 39.086, "p95_ms": 35.187}, "gstreamer_push": {"avg_ms": 1.1, "count": 205, "max_ms": 1.292, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.629, "count": 205, "max_ms": 42.334, "p95_ms": 33.809}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.384, "count": 205, "max_ms": 90.128, "p95_ms": 82.24}, "yolo11_640_fire_11class": {"avg_ms": 48.786, "count": 205, "max_ms": 67.111, "p95_ms": 62.724}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.313, "count": 205, "max_ms": 83.993, "p95_ms": 77.631}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.367, "count": 205, "max_ms": 161.18, "p95_ms": 153.237}, "read_interval": {"avg_ms": 146.457, "count": 205, "max_ms": 173.128, "p95_ms": 165.267}, "triton_wait": {"avg_ms": 63.604, "count": 205, "max_ms": 90.14, "p95_ms": 82.252}, "visualize": {"avg_ms": 4.303, "count": 205, "max_ms": 5.498, "p95_ms": 4.632}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11964}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11025.027, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:08:27+09:00", "last_error_summary": "[h264 @ 0x55fb6efb26c0] error while decoding MB 77 37, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 481, "h264_decode_error_count": 1417, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:08:27+09:00", "last_error_summary": "[h264 @ 0x55fb6efb26c0] error while decoding MB 77 37, bytestream -5", "last_exit_at": "2026-07-01T10:08:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.062, "last_started_at": "2026-07-01T10:05:47+09:00", "pid": 95705, "process": {"exists": false, "pid": 95705}, "restart_count": 528, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 529}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.03, "sample_count": 1960, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1435761, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:08:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T10:08:27+09:00", "last_frame_read_age_sec": 0.174, "last_frame_read_at": "2026-07-01T10:08:27+09:00", "mjpeg": {"age_sec": 0.165, "exists": true, "mtime": "2026-07-01T10:08:27.178290+09:00", "mtime_epoch": 1782868107.1782901, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151478}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:08:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5050.336, "name": "celery", "pid": 267, "rss_mb": 5050.336, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.861, "pushed": 6.861, "read": 6.828}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.019, "count": 205, "max_ms": 47.356, "p95_ms": 35.238}, "gstreamer_push": {"avg_ms": 1.101, "count": 205, "max_ms": 1.292, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.654, "count": 205, "max_ms": 42.334, "p95_ms": 33.809}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.513, "count": 205, "max_ms": 90.128, "p95_ms": 82.338}, "yolo11_640_fire_11class": {"avg_ms": 48.891, "count": 205, "max_ms": 67.111, "p95_ms": 63.122}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.469, "count": 205, "max_ms": 83.993, "p95_ms": 78.169}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.446, "count": 205, "max_ms": 168.017, "p95_ms": 153.374}, "read_interval": {"avg_ms": 146.38, "count": 204, "max_ms": 173.128, "p95_ms": 165.267}, "triton_wait": {"avg_ms": 63.732, "count": 205, "max_ms": 90.14, "p95_ms": 82.35}, "visualize": {"avg_ms": 4.3, "count": 205, "max_ms": 5.498, "p95_ms": 4.619}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11964}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11025.02, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 481, "h264_decode_error_count": 1417, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:08:27+09:00", "last_error_summary": "[h264 @ 0x55fb6efb26c0] error while decoding MB 77 37, bytestream -5", "last_exit_at": "2026-07-01T10:08:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.067, "last_started_at": "2026-07-01T10:05:47+09:00", "pid": 95705, "process": {"exists": false, "pid": 95705}, "restart_count": 528, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 529}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.03, "sample_count": 1960, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1435761, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:08:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T10:08:27+09:00", "last_frame_read_age_sec": 0.18, "last_frame_read_at": "2026-07-01T10:08:27+09:00", "mjpeg": {"age_sec": 0.171, "exists": true, "mtime": "2026-07-01T10:08:27.178290+09:00", "mtime_epoch": 1782868107.1782901, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151478}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:08:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5050.336, "name": "celery", "pid": 267, "rss_mb": 5050.336, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.86, "pushed": 6.86, "read": 6.827}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.019, "count": 205, "max_ms": 47.356, "p95_ms": 35.238}, "gstreamer_push": {"avg_ms": 1.101, "count": 205, "max_ms": 1.292, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.654, "count": 205, "max_ms": 42.334, "p95_ms": 33.809}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.513, "count": 205, "max_ms": 90.128, "p95_ms": 82.338}, "yolo11_640_fire_11class": {"avg_ms": 48.891, "count": 205, "max_ms": 67.111, "p95_ms": 63.122}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.469, "count": 205, "max_ms": 83.993, "p95_ms": 78.169}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.446, "count": 205, "max_ms": 168.017, "p95_ms": 153.374}, "read_interval": {"avg_ms": 146.38, "count": 204, "max_ms": 173.128, "p95_ms": 165.267}, "triton_wait": {"avg_ms": 63.732, "count": 205, "max_ms": 90.14, "p95_ms": 82.35}, "visualize": {"avg_ms": 4.3, "count": 205, "max_ms": 5.498, "p95_ms": 4.619}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11964}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11034.348, "limit_mb": 8796093022207.996}, "detail": {"pid": 95812, "restart_count": 529, "start_count": 530}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 481, "h264_decode_error_count": 1417, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:08:27+09:00", "last_error_summary": "[h264 @ 0x55fb6efb26c0] error while decoding MB 77 37, bytestream -5", "last_exit_at": "2026-07-01T10:08:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:08:29+09:00", "pid": 95812, "process": {"exists": true, "fd_count": 4, "hwm_mb": 43.391, "name": "ffmpeg", "pid": 95812, "rss_mb": 43.391, "state": "S (sleeping)", "threads": 1}, "restart_count": 529, "restart_count_per_hour": 25, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 530}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.069, "sample_count": 1930, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1435761, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:08:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.599, "last_frame_pushed_at": "2026-07-01T10:08:27+09:00", "last_frame_read_age_sec": 4.769, "last_frame_read_at": "2026-07-01T10:08:27+09:00", "mjpeg": {"age_sec": 4.58, "exists": true, "mtime": "2026-07-01T10:08:27.358293+09:00", "mtime_epoch": 1782868107.3582928, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151395}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:08:31+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5050.336, "name": "celery", "pid": 267, "rss_mb": 5050.336, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.817, "pushed": 5.817, "read": 5.783}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.916, "count": 174, "max_ms": 47.356, "p95_ms": 35.375}, "gstreamer_push": {"avg_ms": 1.097, "count": 174, "max_ms": 1.236, "p95_ms": 1.157}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.808, "count": 174, "max_ms": 42.334, "p95_ms": 34.746}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.395, "count": 174, "max_ms": 90.128, "p95_ms": 82.24}, "yolo11_640_fire_11class": {"avg_ms": 49.065, "count": 174, "max_ms": 66.116, "p95_ms": 63.122}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.731, "count": 174, "max_ms": 83.357, "p95_ms": 78.169}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.217, "count": 174, "max_ms": 168.017, "p95_ms": 153.237}, "read_interval": {"avg_ms": 146.117, "count": 173, "max_ms": 172.221, "p95_ms": 165.241}, "triton_wait": {"avg_ms": 63.652, "count": 174, "max_ms": 90.14, "p95_ms": 82.252}, "visualize": {"avg_ms": 4.283, "count": 174, "max_ms": 5.086, "p95_ms": 4.55}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11013.805, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563a58906b80] error while decoding MB 26 32, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 482, "h264_decode_error_count": 1418, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:11:16+09:00", "last_error_summary": "[h264 @ 0x563a58906b80] error while decoding MB 26 32, bytestream -9", "last_exit_at": "2026-07-01T10:08:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.053, "last_started_at": "2026-07-01T10:08:29+09:00", "pid": 95812, "process": {"exists": false, "pid": 95812}, "restart_count": 529, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 530}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.793, "sample_count": 1950, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1436845, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:11:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T10:11:16+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T10:11:16+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T10:11:16.944834+09:00", "mtime_epoch": 1782868276.944834, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149909}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:11:16+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5050.852, "name": "celery", "pid": 267, "rss_mb": 5050.852, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.785, "pushed": 6.785, "read": 6.785}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.202, "count": 203, "max_ms": 46.913, "p95_ms": 36.423}, "gstreamer_push": {"avg_ms": 1.103, "count": 203, "max_ms": 1.745, "p95_ms": 1.168}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.052, "count": 203, "max_ms": 38.128, "p95_ms": 34.027}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.996, "count": 203, "max_ms": 89.235, "p95_ms": 84.213}, "yolo11_640_fire_11class": {"avg_ms": 47.601, "count": 203, "max_ms": 66.206, "p95_ms": 62.59}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.829, "count": 203, "max_ms": 84.374, "p95_ms": 78.091}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.683, "count": 203, "max_ms": 163.105, "p95_ms": 154.879}, "read_interval": {"avg_ms": 147.79, "count": 203, "max_ms": 176.345, "p95_ms": 167.336}, "triton_wait": {"avg_ms": 64.267, "count": 203, "max_ms": 89.247, "p95_ms": 84.26}, "visualize": {"avg_ms": 4.334, "count": 203, "max_ms": 6.292, "p95_ms": 4.788}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11012.82, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:11:16+09:00", "last_error_summary": "[h264 @ 0x563a58906b80] error while decoding MB 26 32, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 482, "h264_decode_error_count": 1418, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:11:16+09:00", "last_error_summary": "[h264 @ 0x563a58906b80] error while decoding MB 26 32, bytestream -9", "last_exit_at": "2026-07-01T10:11:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.136, "last_started_at": "2026-07-01T10:08:29+09:00", "pid": 95812, "process": {"exists": false, "pid": 95812}, "restart_count": 529, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 530}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.793, "sample_count": 1950, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1436846, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:11:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T10:11:17+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-01T10:11:16+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-01T10:11:16.944834+09:00", "mtime_epoch": 1782868276.944834, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149909}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:11:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5050.852, "name": "celery", "pid": 267, "rss_mb": 5050.852, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.769, "pushed": 6.769, "read": 6.769}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.184, "count": 203, "max_ms": 46.913, "p95_ms": 36.423}, "gstreamer_push": {"avg_ms": 1.103, "count": 203, "max_ms": 1.745, "p95_ms": 1.168}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.053, "count": 203, "max_ms": 38.128, "p95_ms": 34.027}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.009, "count": 203, "max_ms": 89.235, "p95_ms": 84.213}, "yolo11_640_fire_11class": {"avg_ms": 47.577, "count": 203, "max_ms": 66.206, "p95_ms": 62.59}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.806, "count": 203, "max_ms": 84.374, "p95_ms": 78.091}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.662, "count": 203, "max_ms": 163.105, "p95_ms": 154.879}, "read_interval": {"avg_ms": 147.79, "count": 203, "max_ms": 176.345, "p95_ms": 167.336}, "triton_wait": {"avg_ms": 64.28, "count": 203, "max_ms": 89.247, "p95_ms": 84.26}, "visualize": {"avg_ms": 4.332, "count": 203, "max_ms": 6.292, "p95_ms": 4.788}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11012.812, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 482, "h264_decode_error_count": 1418, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:11:16+09:00", "last_error_summary": "[h264 @ 0x563a58906b80] error while decoding MB 26 32, bytestream -9", "last_exit_at": "2026-07-01T10:11:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 167.141, "last_started_at": "2026-07-01T10:08:29+09:00", "pid": 95812, "process": {"exists": false, "pid": 95812}, "restart_count": 529, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 530}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.793, "sample_count": 1950, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1436846, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:11:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T10:11:17+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-01T10:11:16+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-01T10:11:16.944834+09:00", "mtime_epoch": 1782868276.944834, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149909}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:11:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5050.852, "name": "celery", "pid": 267, "rss_mb": 5050.852, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.767, "pushed": 6.767, "read": 6.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.184, "count": 203, "max_ms": 46.913, "p95_ms": 36.423}, "gstreamer_push": {"avg_ms": 1.103, "count": 203, "max_ms": 1.745, "p95_ms": 1.168}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.053, "count": 203, "max_ms": 38.128, "p95_ms": 34.027}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.009, "count": 203, "max_ms": 89.235, "p95_ms": 84.213}, "yolo11_640_fire_11class": {"avg_ms": 47.577, "count": 203, "max_ms": 66.206, "p95_ms": 62.59}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.806, "count": 203, "max_ms": 84.374, "p95_ms": 78.091}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.662, "count": 203, "max_ms": 163.105, "p95_ms": 154.879}, "read_interval": {"avg_ms": 147.79, "count": 203, "max_ms": 176.345, "p95_ms": 167.336}, "triton_wait": {"avg_ms": 64.28, "count": 203, "max_ms": 89.247, "p95_ms": 84.26}, "visualize": {"avg_ms": 4.332, "count": 203, "max_ms": 6.292, "p95_ms": 4.788}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11120.605, "limit_mb": 8796093022207.996}, "detail": {"pid": 95919, "restart_count": 530, "start_count": 531}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 482, "h264_decode_error_count": 1418, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:11:16+09:00", "last_error_summary": "[h264 @ 0x563a58906b80] error while decoding MB 26 32, bytestream -9", "last_exit_at": "2026-07-01T10:11:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:11:19+09:00", "pid": 95919, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.18, "name": "ffmpeg", "pid": 95919, "rss_mb": 139.18, "state": "S (sleeping)", "threads": 106}, "restart_count": 530, "restart_count_per_hour": 25, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 531}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.802, "sample_count": 1922, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1436846, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:11:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.117, "last_frame_pushed_at": "2026-07-01T10:11:17+09:00", "last_frame_read_age_sec": 4.248, "last_frame_read_at": "2026-07-01T10:11:16+09:00", "mjpeg": {"age_sec": 4.101, "exists": true, "mtime": "2026-07-01T10:11:17.084836+09:00", "mtime_epoch": 1782868277.0848362, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149000}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:11:21+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5050.852, "name": "celery", "pid": 267, "rss_mb": 5050.852, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 176, "pushed_frames": 176, "read_frames": 175, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.882, "pushed": 5.882, "read": 5.849}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.197, "count": 176, "max_ms": 40.809, "p95_ms": 36.901}, "gstreamer_push": {"avg_ms": 1.105, "count": 176, "max_ms": 1.745, "p95_ms": 1.177}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.015, "count": 176, "max_ms": 38.128, "p95_ms": 34.027}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.685, "count": 176, "max_ms": 89.235, "p95_ms": 84.878}, "yolo11_640_fire_11class": {"avg_ms": 47.267, "count": 176, "max_ms": 66.206, "p95_ms": 62.137}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.315, "count": 176, "max_ms": 84.374, "p95_ms": 77.662}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.286, "count": 176, "max_ms": 163.105, "p95_ms": 155.242}, "read_interval": {"avg_ms": 147.433, "count": 175, "max_ms": 176.345, "p95_ms": 168.072}, "triton_wait": {"avg_ms": 63.969, "count": 176, "max_ms": 89.247, "p95_ms": 84.89}, "visualize": {"avg_ms": 4.315, "count": 176, "max_ms": 6.292, "p95_ms": 4.709}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11982}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11015.355, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b6b6476900] error while decoding MB 7 38, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 483, "h264_decode_error_count": 1419, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:14:02+09:00", "last_error_summary": "[h264 @ 0x55b6b6476900] error while decoding MB 7 38, bytestream -5", "last_exit_at": "2026-07-01T10:11:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.766, "last_started_at": "2026-07-01T10:11:19+09:00", "pid": 95919, "process": {"exists": false, "pid": 95919}, "restart_count": 530, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 531}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.316, "sample_count": 1995, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1437940, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:14:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-01T10:14:02+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T10:14:02+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T10:14:02.903351+09:00", "mtime_epoch": 1782868442.9033508, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 150068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:14:02+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5051.367, "name": "celery", "pid": 267, "rss_mb": 5051.367, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.709, "pushed": 6.709, "read": 6.709}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.254, "count": 201, "max_ms": 40.253, "p95_ms": 36.738}, "gstreamer_push": {"avg_ms": 1.085, "count": 201, "max_ms": 1.516, "p95_ms": 1.261}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.161, "count": 201, "max_ms": 41.152, "p95_ms": 36.059}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.051, "count": 201, "max_ms": 90.768, "p95_ms": 88.182}, "yolo11_640_fire_11class": {"avg_ms": 49.079, "count": 201, "max_ms": 66.466, "p95_ms": 63.896}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.215, "count": 201, "max_ms": 87.142, "p95_ms": 78.04}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.364, "count": 201, "max_ms": 162.685, "p95_ms": 158.951}, "read_interval": {"avg_ms": 149.427, "count": 201, "max_ms": 175.06, "p95_ms": 172.493}, "triton_wait": {"avg_ms": 66.315, "count": 201, "max_ms": 90.781, "p95_ms": 88.195}, "visualize": {"avg_ms": 4.309, "count": 201, "max_ms": 6.504, "p95_ms": 4.948}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11982}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11014.906, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:14:02+09:00", "last_error_summary": "[h264 @ 0x55b6b6476900] error while decoding MB 7 38, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 483, "h264_decode_error_count": 1419, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:14:02+09:00", "last_error_summary": "[h264 @ 0x55b6b6476900] error while decoding MB 7 38, bytestream -5", "last_exit_at": "2026-07-01T10:14:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.856, "last_started_at": "2026-07-01T10:11:19+09:00", "pid": 95919, "process": {"exists": false, "pid": 95919}, "restart_count": 530, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 531}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.316, "sample_count": 1995, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1437941, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:14:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T10:14:03+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T10:14:02+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-01T10:14:02.903351+09:00", "mtime_epoch": 1782868442.9033508, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 150068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:14:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5051.367, "name": "celery", "pid": 267, "rss_mb": 5051.367, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.72, "pushed": 6.72, "read": 6.687}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.273, "count": 201, "max_ms": 40.253, "p95_ms": 36.931}, "gstreamer_push": {"avg_ms": 1.085, "count": 201, "max_ms": 1.516, "p95_ms": 1.261}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.155, "count": 201, "max_ms": 41.152, "p95_ms": 36.059}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.066, "count": 201, "max_ms": 90.768, "p95_ms": 88.182}, "yolo11_640_fire_11class": {"avg_ms": 49.069, "count": 201, "max_ms": 66.466, "p95_ms": 63.896}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.222, "count": 201, "max_ms": 87.142, "p95_ms": 78.04}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.4, "count": 201, "max_ms": 162.685, "p95_ms": 158.951}, "read_interval": {"avg_ms": 149.464, "count": 200, "max_ms": 175.06, "p95_ms": 172.493}, "triton_wait": {"avg_ms": 66.331, "count": 201, "max_ms": 90.781, "p95_ms": 88.195}, "visualize": {"avg_ms": 4.31, "count": 201, "max_ms": 6.504, "p95_ms": 4.948}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11982}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11014.965, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 483, "h264_decode_error_count": 1419, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:14:02+09:00", "last_error_summary": "[h264 @ 0x55b6b6476900] error while decoding MB 7 38, bytestream -5", "last_exit_at": "2026-07-01T10:14:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.862, "last_started_at": "2026-07-01T10:11:19+09:00", "pid": 95919, "process": {"exists": false, "pid": 95919}, "restart_count": 530, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 531}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.316, "sample_count": 1995, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1437941, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:14:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T10:14:03+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-01T10:14:02+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-01T10:14:02.903351+09:00", "mtime_epoch": 1782868442.9033508, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 150068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:14:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5051.367, "name": "celery", "pid": 267, "rss_mb": 5051.367, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.719, "pushed": 6.719, "read": 6.686}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.273, "count": 201, "max_ms": 40.253, "p95_ms": 36.931}, "gstreamer_push": {"avg_ms": 1.085, "count": 201, "max_ms": 1.516, "p95_ms": 1.261}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.155, "count": 201, "max_ms": 41.152, "p95_ms": 36.059}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.066, "count": 201, "max_ms": 90.768, "p95_ms": 88.182}, "yolo11_640_fire_11class": {"avg_ms": 49.069, "count": 201, "max_ms": 66.466, "p95_ms": 63.896}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.222, "count": 201, "max_ms": 87.142, "p95_ms": 78.04}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.4, "count": 201, "max_ms": 162.685, "p95_ms": 158.951}, "read_interval": {"avg_ms": 149.464, "count": 200, "max_ms": 175.06, "p95_ms": 172.493}, "triton_wait": {"avg_ms": 66.331, "count": 201, "max_ms": 90.781, "p95_ms": 88.195}, "visualize": {"avg_ms": 4.31, "count": 201, "max_ms": 6.504, "p95_ms": 4.948}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11982}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11122.18, "limit_mb": 8796093022207.996}, "detail": {"pid": 96026, "restart_count": 531, "start_count": 532}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 483, "h264_decode_error_count": 1419, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:14:02+09:00", "last_error_summary": "[h264 @ 0x55b6b6476900] error while decoding MB 7 38, bytestream -5", "last_exit_at": "2026-07-01T10:14:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:14:05+09:00", "pid": 96026, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.219, "name": "ffmpeg", "pid": 96026, "rss_mb": 139.219, "state": "S (sleeping)", "threads": 106}, "restart_count": 531, "restart_count_per_hour": 25, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 532}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.357, "sample_count": 1966, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1437941, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:14:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.088, "last_frame_pushed_at": "2026-07-01T10:14:03+09:00", "last_frame_read_age_sec": 4.226, "last_frame_read_at": "2026-07-01T10:14:02+09:00", "mjpeg": {"age_sec": 4.069, "exists": true, "mtime": "2026-07-01T10:14:03.055353+09:00", "mtime_epoch": 1782868443.0553532, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149864}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:14:07+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5051.367, "name": "celery", "pid": 267, "rss_mb": 5051.367, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 173, "pushed_frames": 173, "read_frames": 172, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.788, "pushed": 5.788, "read": 5.754}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.459, "count": 173, "max_ms": 40.253, "p95_ms": 37.669}, "gstreamer_push": {"avg_ms": 1.093, "count": 173, "max_ms": 1.516, "p95_ms": 1.288}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.462, "count": 173, "max_ms": 41.152, "p95_ms": 37.282}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.441, "count": 173, "max_ms": 90.768, "p95_ms": 88.375}, "yolo11_640_fire_11class": {"avg_ms": 49.787, "count": 173, "max_ms": 66.466, "p95_ms": 64.087}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.447, "count": 173, "max_ms": 87.142, "p95_ms": 78.738}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.789, "count": 173, "max_ms": 162.685, "p95_ms": 159.002}, "read_interval": {"avg_ms": 149.971, "count": 172, "max_ms": 175.06, "p95_ms": 173.262}, "triton_wait": {"avg_ms": 66.651, "count": 173, "max_ms": 90.781, "p95_ms": 88.391}, "visualize": {"avg_ms": 4.297, "count": 173, "max_ms": 6.504, "p95_ms": 4.948}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11991}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11016.5, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5629819ae200] error while decoding MB 4 22, bytestream -17", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 484, "h264_decode_error_count": 1420, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:16:48+09:00", "last_error_summary": "[h264 @ 0x5629819ae200] error while decoding MB 4 22, bytestream -17", "last_exit_at": "2026-07-01T10:14:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.303, "last_started_at": "2026-07-01T10:14:05+09:00", "pid": 96026, "process": {"exists": false, "pid": 96026}, "restart_count": 531, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 532}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.039, "sample_count": 1989, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1439019, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:16:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T10:16:48+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T10:16:48+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-01T10:16:48.377882+09:00", "mtime_epoch": 1782868608.3778815, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157808}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:16:48+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5051.883, "name": "celery", "pid": 267, "rss_mb": 5051.883, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.681, "pushed": 6.681, "read": 6.681}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.91, "count": 200, "max_ms": 42.919, "p95_ms": 36.762}, "gstreamer_push": {"avg_ms": 1.101, "count": 200, "max_ms": 1.377, "p95_ms": 1.16}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.565, "count": 200, "max_ms": 37.981, "p95_ms": 33.895}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.372, "count": 200, "max_ms": 90.25, "p95_ms": 85.003}, "yolo11_640_fire_11class": {"avg_ms": 49.598, "count": 200, "max_ms": 67.691, "p95_ms": 63.452}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.31, "count": 200, "max_ms": 84.978, "p95_ms": 79.384}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.728, "count": 200, "max_ms": 159.076, "p95_ms": 155.849}, "read_interval": {"avg_ms": 150.107, "count": 200, "max_ms": 174.103, "p95_ms": 168.974}, "triton_wait": {"avg_ms": 66.674, "count": 200, "max_ms": 90.263, "p95_ms": 85.015}, "visualize": {"avg_ms": 4.282, "count": 200, "max_ms": 5.405, "p95_ms": 4.527}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11991}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11016.309, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:16:48+09:00", "last_error_summary": "[h264 @ 0x5629819ae200] error while decoding MB 4 22, bytestream -17", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 484, "h264_decode_error_count": 1420, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:16:48+09:00", "last_error_summary": "[h264 @ 0x5629819ae200] error while decoding MB 4 22, bytestream -17", "last_exit_at": "2026-07-01T10:16:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.397, "last_started_at": "2026-07-01T10:14:05+09:00", "pid": 96026, "process": {"exists": false, "pid": 96026}, "restart_count": 531, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 532}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.039, "sample_count": 1990, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1439020, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:16:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T10:16:48+09:00", "last_frame_read_age_sec": 0.146, "last_frame_read_at": "2026-07-01T10:16:48+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-01T10:16:48.377882+09:00", "mtime_epoch": 1782868608.3778815, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157808}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:16:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5051.883, "name": "celery", "pid": 267, "rss_mb": 5051.883, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.691, "pushed": 6.691, "read": 6.657}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.916, "count": 200, "max_ms": 42.919, "p95_ms": 36.762}, "gstreamer_push": {"avg_ms": 1.102, "count": 200, "max_ms": 1.377, "p95_ms": 1.16}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.573, "count": 200, "max_ms": 37.981, "p95_ms": 33.895}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.337, "count": 200, "max_ms": 90.25, "p95_ms": 85.003}, "yolo11_640_fire_11class": {"avg_ms": 49.54, "count": 200, "max_ms": 67.691, "p95_ms": 63.452}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.245, "count": 200, "max_ms": 84.978, "p95_ms": 79.384}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.727, "count": 200, "max_ms": 159.076, "p95_ms": 155.849}, "read_interval": {"avg_ms": 150.098, "count": 199, "max_ms": 174.103, "p95_ms": 169.398}, "triton_wait": {"avg_ms": 66.639, "count": 200, "max_ms": 90.263, "p95_ms": 85.015}, "visualize": {"avg_ms": 4.285, "count": 200, "max_ms": 5.405, "p95_ms": 4.682}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11991}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11016.219, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 484, "h264_decode_error_count": 1420, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:16:48+09:00", "last_error_summary": "[h264 @ 0x5629819ae200] error while decoding MB 4 22, bytestream -17", "last_exit_at": "2026-07-01T10:16:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.402, "last_started_at": "2026-07-01T10:14:05+09:00", "pid": 96026, "process": {"exists": false, "pid": 96026}, "restart_count": 531, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 532}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.039, "sample_count": 1990, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1439020, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:16:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T10:16:48+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-01T10:16:48+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-01T10:16:48.377882+09:00", "mtime_epoch": 1782868608.3778815, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157808}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:16:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5051.883, "name": "celery", "pid": 267, "rss_mb": 5051.883, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.689, "pushed": 6.689, "read": 6.656}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.916, "count": 200, "max_ms": 42.919, "p95_ms": 36.762}, "gstreamer_push": {"avg_ms": 1.102, "count": 200, "max_ms": 1.377, "p95_ms": 1.16}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.573, "count": 200, "max_ms": 37.981, "p95_ms": 33.895}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.337, "count": 200, "max_ms": 90.25, "p95_ms": 85.003}, "yolo11_640_fire_11class": {"avg_ms": 49.54, "count": 200, "max_ms": 67.691, "p95_ms": 63.452}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.245, "count": 200, "max_ms": 84.978, "p95_ms": 79.384}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.727, "count": 200, "max_ms": 159.076, "p95_ms": 155.849}, "read_interval": {"avg_ms": 150.098, "count": 199, "max_ms": 174.103, "p95_ms": 169.398}, "triton_wait": {"avg_ms": 66.639, "count": 200, "max_ms": 90.263, "p95_ms": 85.015}, "visualize": {"avg_ms": 4.285, "count": 200, "max_ms": 5.405, "p95_ms": 4.682}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 11991}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11132.23, "limit_mb": 8796093022207.996}, "detail": {"pid": 96133, "restart_count": 532, "start_count": 533}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 484, "h264_decode_error_count": 1420, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:16:48+09:00", "last_error_summary": "[h264 @ 0x5629819ae200] error while decoding MB 4 22, bytestream -17", "last_exit_at": "2026-07-01T10:16:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:16:51+09:00", "pid": 96133, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.27, "name": "ffmpeg", "pid": 96133, "rss_mb": 147.27, "state": "S (sleeping)", "threads": 106}, "restart_count": 532, "restart_count_per_hour": 25, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 533}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.129, "sample_count": 1957, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1439020, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:16:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.897, "last_frame_pushed_at": "2026-07-01T10:16:48+09:00", "last_frame_read_age_sec": 5.039, "last_frame_read_at": "2026-07-01T10:16:48+09:00", "mjpeg": {"age_sec": 4.883, "exists": true, "mtime": "2026-07-01T10:16:48.529884+09:00", "mtime_epoch": 1782868608.5298839, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157923}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:16:53+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5051.883, "name": "celery", "pid": 267, "rss_mb": 5051.883, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 168, "pushed_frames": 168, "read_frames": 167, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.624, "pushed": 5.624, "read": 5.591}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.872, "count": 168, "max_ms": 42.919, "p95_ms": 36.75}, "gstreamer_push": {"avg_ms": 1.096, "count": 168, "max_ms": 1.333, "p95_ms": 1.156}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.59, "count": 168, "max_ms": 37.981, "p95_ms": 33.978}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.96, "count": 168, "max_ms": 90.25, "p95_ms": 84.901}, "yolo11_640_fire_11class": {"avg_ms": 49.051, "count": 168, "max_ms": 65.361, "p95_ms": 62.497}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.92, "count": 168, "max_ms": 84.978, "p95_ms": 79.316}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.297, "count": 168, "max_ms": 159.076, "p95_ms": 154.952}, "read_interval": {"avg_ms": 149.588, "count": 167, "max_ms": 172.225, "p95_ms": 167.257}, "triton_wait": {"avg_ms": 66.301, "count": 168, "max_ms": 90.263, "p95_ms": 84.989}, "visualize": {"avg_ms": 4.267, "count": 168, "max_ms": 5.201, "p95_ms": 4.523}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13632}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10961.613, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55faf0ff6200] missing picture in access unit with size 18 [h264 @ 0x55faf10848c0] no frame! [in#0/rtsp @ 0x55faf0ff1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faf0ff1e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55faf0ff6840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 211, "h264_decode_error_count": 217, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:18:58+09:00", "last_error_summary": "[NULL @ 0x55faf0ff6200] missing picture in access unit with size 18 [h264 @ 0x55faf10848c0] no frame! [in#0/rtsp @ 0x55faf0ff1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faf0ff1e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55faf0ff6840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T10:05:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 804.186, "last_started_at": "2026-07-01T10:05:34+09:00", "pid": 95553, "process": {"exists": false, "pid": 95553}, "restart_count": 210, "restart_count_per_hour": 6, "rtsp_timeout_count": 348, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 211}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.545, "sample_count": 2572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1635947, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:18:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-01T10:18:58+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T10:18:58+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T10:18:58.511883+09:00", "mtime_epoch": 1782868738.5118825, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 210031}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:18:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1962.633, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.68, "pushed": 0.68, "read": 0.737}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.033, "count": 12, "max_ms": 25.182, "p95_ms": 25.182}, "gstreamer_push": {"avg_ms": 1.217, "count": 12, "max_ms": 1.461, "p95_ms": 1.461}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.882, "count": 12, "max_ms": 56.962, "p95_ms": 56.962}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.562, "count": 12, "max_ms": 82.805, "p95_ms": 82.805}, "yolo11_640_fire_11class": {"avg_ms": 50.254, "count": 12, "max_ms": 60.68, "p95_ms": 60.68}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.412, "count": 12, "max_ms": 79.894, "p95_ms": 79.894}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.967, "count": 12, "max_ms": 112.015, "p95_ms": 112.015}, "read_interval": {"avg_ms": 2710.776, "count": 13, "max_ms": 17652.465, "p95_ms": 17652.465}, "triton_wait": {"avg_ms": 59.644, "count": 12, "max_ms": 82.82, "p95_ms": 82.82}, "visualize": {"avg_ms": 3.509, "count": 12, "max_ms": 4.535, "p95_ms": 4.535}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.643, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13632}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10961.09, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:18:58+09:00", "last_error_summary": "[NULL @ 0x55faf0ff6200] missing picture in access unit with size 18 [h264 @ 0x55faf10848c0] no frame! [in#0/rtsp @ 0x55faf0ff1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faf0ff1e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55faf0ff6840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 211, "h264_decode_error_count": 217, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:18:58+09:00", "last_error_summary": "[NULL @ 0x55faf0ff6200] missing picture in access unit with size 18 [h264 @ 0x55faf10848c0] no frame! [in#0/rtsp @ 0x55faf0ff1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faf0ff1e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55faf0ff6840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T10:18:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 804.243, "last_started_at": "2026-07-01T10:05:34+09:00", "pid": 95553, "process": {"exists": false, "pid": 95553}, "restart_count": 210, "restart_count_per_hour": 6, "rtsp_timeout_count": 348, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 211}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.545, "sample_count": 2572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1635948, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:18:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T10:18:58+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-01T10:18:58+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-01T10:18:58.511883+09:00", "mtime_epoch": 1782868738.5118825, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 210031}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:18:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1962.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.734, "pushed": 0.734, "read": 0.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.966, "count": 13, "max_ms": 25.182, "p95_ms": 25.182}, "gstreamer_push": {"avg_ms": 1.219, "count": 13, "max_ms": 1.461, "p95_ms": 1.461}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.888, "count": 13, "max_ms": 56.962, "p95_ms": 56.962}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.773, "count": 13, "max_ms": 82.805, "p95_ms": 82.805}, "yolo11_640_fire_11class": {"avg_ms": 50.804, "count": 13, "max_ms": 60.68, "p95_ms": 60.68}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.371, "count": 13, "max_ms": 79.894, "p95_ms": 79.894}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.129, "count": 13, "max_ms": 112.015, "p95_ms": 112.015}, "read_interval": {"avg_ms": 2710.776, "count": 13, "max_ms": 17652.465, "p95_ms": 17652.465}, "triton_wait": {"avg_ms": 60.849, "count": 13, "max_ms": 82.82, "p95_ms": 82.82}, "visualize": {"avg_ms": 3.505, "count": 13, "max_ms": 4.535, "p95_ms": 4.535}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.699, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13632}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10961.082, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 211, "h264_decode_error_count": 217, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:18:58+09:00", "last_error_summary": "[NULL @ 0x55faf0ff6200] missing picture in access unit with size 18 [h264 @ 0x55faf10848c0] no frame! [in#0/rtsp @ 0x55faf0ff1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faf0ff1e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55faf0ff6840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T10:18:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 804.249, "last_started_at": "2026-07-01T10:05:34+09:00", "pid": 95553, "process": {"exists": false, "pid": 95553}, "restart_count": 210, "restart_count_per_hour": 6, "rtsp_timeout_count": 348, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 211}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.545, "sample_count": 2572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1635948, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:18:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T10:18:58+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-01T10:18:58+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T10:18:58.511883+09:00", "mtime_epoch": 1782868738.5118825, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 210031}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:18:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1962.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.734, "pushed": 0.734, "read": 0.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.966, "count": 13, "max_ms": 25.182, "p95_ms": 25.182}, "gstreamer_push": {"avg_ms": 1.219, "count": 13, "max_ms": 1.461, "p95_ms": 1.461}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.888, "count": 13, "max_ms": 56.962, "p95_ms": 56.962}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.773, "count": 13, "max_ms": 82.805, "p95_ms": 82.805}, "yolo11_640_fire_11class": {"avg_ms": 50.804, "count": 13, "max_ms": 60.68, "p95_ms": 60.68}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.371, "count": 13, "max_ms": 79.894, "p95_ms": 79.894}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.129, "count": 13, "max_ms": 112.015, "p95_ms": 112.015}, "read_interval": {"avg_ms": 2710.776, "count": 13, "max_ms": 17652.465, "p95_ms": 17652.465}, "triton_wait": {"avg_ms": 60.849, "count": 13, "max_ms": 82.82, "p95_ms": 82.82}, "visualize": {"avg_ms": 3.505, "count": 13, "max_ms": 4.535, "p95_ms": 4.535}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.706, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13632}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 11140.988, "limit_mb": 8796093022207.996}, "detail": {"pid": 96240, "restart_count": 211, "start_count": 212}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 211, "h264_decode_error_count": 217, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:18:58+09:00", "last_error_summary": "[NULL @ 0x55faf0ff6200] missing picture in access unit with size 18 [h264 @ 0x55faf10848c0] no frame! [in#0/rtsp @ 0x55faf0ff1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faf0ff1e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55faf0ff6840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T10:19:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:19:00+09:00", "pid": 96240, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.578, "name": "ffmpeg", "pid": 96240, "rss_mb": 209.578, "state": "S (sleeping)", "threads": 106}, "restart_count": 211, "restart_count_per_hour": 7, "rtsp_timeout_count": 348, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 212}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.522, "sample_count": 2532, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1635948, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:18:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.221, "last_frame_pushed_at": "2026-07-01T10:18:58+09:00", "last_frame_read_age_sec": 4.327, "last_frame_read_at": "2026-07-01T10:18:58+09:00", "mjpeg": {"age_sec": 4.202, "exists": true, "mtime": "2026-07-01T10:18:58.627884+09:00", "mtime_epoch": 1782868738.6278844, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 209692}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:19:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1962.633, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.593, "pushed": 0.593, "read": 0.593}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.966, "count": 13, "max_ms": 25.182, "p95_ms": 25.182}, "gstreamer_push": {"avg_ms": 1.219, "count": 13, "max_ms": 1.461, "p95_ms": 1.461}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.888, "count": 13, "max_ms": 56.962, "p95_ms": 56.962}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.773, "count": 13, "max_ms": 82.805, "p95_ms": 82.805}, "yolo11_640_fire_11class": {"avg_ms": 50.804, "count": 13, "max_ms": 60.68, "p95_ms": 60.68}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.371, "count": 13, "max_ms": 79.894, "p95_ms": 79.894}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.129, "count": 13, "max_ms": 112.015, "p95_ms": 112.015}, "read_interval": {"avg_ms": 2710.776, "count": 13, "max_ms": 17652.465, "p95_ms": 17652.465}, "triton_wait": {"avg_ms": 60.849, "count": 13, "max_ms": 82.82, "p95_ms": 82.82}, "visualize": {"avg_ms": 3.505, "count": 13, "max_ms": 4.535, "p95_ms": 4.535}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.915, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12001}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11020.445, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c354fc8840] error while decoding MB 70 32, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 485, "h264_decode_error_count": 1421, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:19:34+09:00", "last_error_summary": "[h264 @ 0x55c354fc8840] error while decoding MB 70 32, bytestream -9", "last_exit_at": "2026-07-01T10:16:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.464, "last_started_at": "2026-07-01T10:16:51+09:00", "pid": 96133, "process": {"exists": false, "pid": 96133}, "restart_count": 532, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 533}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.907, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1440138, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:19:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T10:19:34+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T10:19:34+09:00", "mjpeg": {"age_sec": 0.035, "exists": true, "mtime": "2026-07-01T10:19:34.840443+09:00", "mtime_epoch": 1782868774.8404427, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161413}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:19:34+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5052.398, "name": "celery", "pid": 267, "rss_mb": 5052.398, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.709, "pushed": 6.709, "read": 6.709}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.418, "count": 201, "max_ms": 47.891, "p95_ms": 35.899}, "gstreamer_push": {"avg_ms": 1.113, "count": 201, "max_ms": 1.507, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.025, "count": 201, "max_ms": 40.833, "p95_ms": 36.733}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.61, "count": 201, "max_ms": 88.59, "p95_ms": 85.54}, "yolo11_640_fire_11class": {"avg_ms": 49.355, "count": 201, "max_ms": 67.939, "p95_ms": 64.14}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.942, "count": 201, "max_ms": 83.947, "p95_ms": 79.956}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.276, "count": 201, "max_ms": 300.81, "p95_ms": 157.306}, "read_interval": {"avg_ms": 149.478, "count": 201, "max_ms": 311.054, "p95_ms": 170.822}, "triton_wait": {"avg_ms": 64.812, "count": 201, "max_ms": 88.602, "p95_ms": 85.551}, "visualize": {"avg_ms": 4.293, "count": 201, "max_ms": 5.443, "p95_ms": 4.729}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12001}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11019.59, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:19:34+09:00", "last_error_summary": "[h264 @ 0x55c354fc8840] error while decoding MB 70 32, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 485, "h264_decode_error_count": 1421, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:19:34+09:00", "last_error_summary": "[h264 @ 0x55c354fc8840] error while decoding MB 70 32, bytestream -9", "last_exit_at": "2026-07-01T10:19:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.564, "last_started_at": "2026-07-01T10:16:51+09:00", "pid": 96133, "process": {"exists": false, "pid": 96133}, "restart_count": 532, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 533}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.907, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1440139, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:19:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T10:19:34+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-01T10:19:34+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T10:19:34.840443+09:00", "mtime_epoch": 1782868774.8404427, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161413}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:19:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5052.398, "name": "celery", "pid": 267, "rss_mb": 5052.398, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.719, "pushed": 6.719, "read": 6.685}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.407, "count": 201, "max_ms": 47.891, "p95_ms": 35.899}, "gstreamer_push": {"avg_ms": 1.113, "count": 201, "max_ms": 1.507, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.051, "count": 201, "max_ms": 40.833, "p95_ms": 36.809}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.649, "count": 201, "max_ms": 88.59, "p95_ms": 85.54}, "yolo11_640_fire_11class": {"avg_ms": 49.374, "count": 201, "max_ms": 67.939, "p95_ms": 64.14}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.992, "count": 201, "max_ms": 83.947, "p95_ms": 79.956}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.319, "count": 201, "max_ms": 300.81, "p95_ms": 157.306}, "read_interval": {"avg_ms": 149.486, "count": 200, "max_ms": 311.054, "p95_ms": 170.822}, "triton_wait": {"avg_ms": 64.85, "count": 201, "max_ms": 88.602, "p95_ms": 85.551}, "visualize": {"avg_ms": 4.293, "count": 201, "max_ms": 5.443, "p95_ms": 4.729}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12001}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11019.582, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 485, "h264_decode_error_count": 1421, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:19:34+09:00", "last_error_summary": "[h264 @ 0x55c354fc8840] error while decoding MB 70 32, bytestream -9", "last_exit_at": "2026-07-01T10:19:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.569, "last_started_at": "2026-07-01T10:16:51+09:00", "pid": 96133, "process": {"exists": false, "pid": 96133}, "restart_count": 532, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 533}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.907, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1440139, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:19:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T10:19:34+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-01T10:19:34+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-01T10:19:34.840443+09:00", "mtime_epoch": 1782868774.8404427, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161413}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:19:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5052.398, "name": "celery", "pid": 267, "rss_mb": 5052.398, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 200, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.717, "pushed": 6.717, "read": 6.684}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.407, "count": 201, "max_ms": 47.891, "p95_ms": 35.899}, "gstreamer_push": {"avg_ms": 1.113, "count": 201, "max_ms": 1.507, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.051, "count": 201, "max_ms": 40.833, "p95_ms": 36.809}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.649, "count": 201, "max_ms": 88.59, "p95_ms": 85.54}, "yolo11_640_fire_11class": {"avg_ms": 49.374, "count": 201, "max_ms": 67.939, "p95_ms": 64.14}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.992, "count": 201, "max_ms": 83.947, "p95_ms": 79.956}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.319, "count": 201, "max_ms": 300.81, "p95_ms": 157.306}, "read_interval": {"avg_ms": 149.486, "count": 200, "max_ms": 311.054, "p95_ms": 170.822}, "triton_wait": {"avg_ms": 64.85, "count": 201, "max_ms": 88.602, "p95_ms": 85.551}, "visualize": {"avg_ms": 4.293, "count": 201, "max_ms": 5.443, "p95_ms": 4.729}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12001}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11127.27, "limit_mb": 8796093022207.996}, "detail": {"pid": 96392, "restart_count": 533, "start_count": 534}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 485, "h264_decode_error_count": 1421, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:19:34+09:00", "last_error_summary": "[h264 @ 0x55c354fc8840] error while decoding MB 70 32, bytestream -9", "last_exit_at": "2026-07-01T10:19:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:19:37+09:00", "pid": 96392, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.469, "name": "ffmpeg", "pid": 96392, "rss_mb": 138.469, "state": "S (sleeping)", "threads": 106}, "restart_count": 533, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 534}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.944, "sample_count": 1979, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1440139, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:19:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.33, "last_frame_pushed_at": "2026-07-01T10:19:34+09:00", "last_frame_read_age_sec": 4.476, "last_frame_read_at": "2026-07-01T10:19:34+09:00", "mjpeg": {"age_sec": 4.317, "exists": true, "mtime": "2026-07-01T10:19:34.984445+09:00", "mtime_epoch": 1782868774.9844449, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161473}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:19:39+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5052.398, "name": "celery", "pid": 267, "rss_mb": 5052.398, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 171, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.756, "pushed": 5.756, "read": 5.723}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.226, "count": 172, "max_ms": 43.45, "p95_ms": 35.39}, "gstreamer_push": {"avg_ms": 1.111, "count": 172, "max_ms": 1.507, "p95_ms": 1.209}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.881, "count": 172, "max_ms": 40.713, "p95_ms": 36.809}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.854, "count": 172, "max_ms": 88.59, "p95_ms": 85.856}, "yolo11_640_fire_11class": {"avg_ms": 49.399, "count": 172, "max_ms": 67.939, "p95_ms": 65.431}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.815, "count": 172, "max_ms": 83.947, "p95_ms": 80.283}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.268, "count": 172, "max_ms": 300.81, "p95_ms": 157.982}, "read_interval": {"avg_ms": 149.408, "count": 171, "max_ms": 311.054, "p95_ms": 170.937}, "triton_wait": {"avg_ms": 64.957, "count": 172, "max_ms": 88.602, "p95_ms": 85.869}, "visualize": {"avg_ms": 4.285, "count": 172, "max_ms": 5.168, "p95_ms": 4.729}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12010}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10895.621, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c348ae18c0] error while decoding MB 21 13, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 486, "h264_decode_error_count": 1422, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:22:21+09:00", "last_error_summary": "[h264 @ 0x55c348ae18c0] error while decoding MB 21 13, bytestream -5", "last_exit_at": "2026-07-01T10:19:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.91, "last_started_at": "2026-07-01T10:19:37+09:00", "pid": 96392, "process": {"exists": false, "pid": 96392}, "restart_count": 533, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 534}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.103, "sample_count": 2023, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1441231, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:22:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T10:22:21+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T10:22:21+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T10:22:21.167254+09:00", "mtime_epoch": 1782868941.1672542, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166294}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:22:21+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5052.914, "name": "celery", "pid": 267, "rss_mb": 5052.914, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 199, "pushed_frames": 199, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.656, "pushed": 6.656, "read": 6.656}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.231, "count": 199, "max_ms": 49.0, "p95_ms": 35.965}, "gstreamer_push": {"avg_ms": 1.093, "count": 199, "max_ms": 1.81, "p95_ms": 1.148}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.358, "count": 199, "max_ms": 40.114, "p95_ms": 34.979}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.614, "count": 199, "max_ms": 91.143, "p95_ms": 84.502}, "yolo11_640_fire_11class": {"avg_ms": 49.29, "count": 199, "max_ms": 71.381, "p95_ms": 64.234}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.975, "count": 199, "max_ms": 85.963, "p95_ms": 80.929}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.657, "count": 199, "max_ms": 179.663, "p95_ms": 159.556}, "read_interval": {"avg_ms": 150.739, "count": 199, "max_ms": 191.938, "p95_ms": 172.106}, "triton_wait": {"avg_ms": 65.142, "count": 199, "max_ms": 91.155, "p95_ms": 84.645}, "visualize": {"avg_ms": 4.396, "count": 199, "max_ms": 6.043, "p95_ms": 5.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12010}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10895.27, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:22:21+09:00", "last_error_summary": "[h264 @ 0x55c348ae18c0] error while decoding MB 21 13, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 486, "h264_decode_error_count": 1422, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:22:21+09:00", "last_error_summary": "[h264 @ 0x55c348ae18c0] error while decoding MB 21 13, bytestream -5", "last_exit_at": "2026-07-01T10:22:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.998, "last_started_at": "2026-07-01T10:19:37+09:00", "pid": 96392, "process": {"exists": false, "pid": 96392}, "restart_count": 533, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 534}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.103, "sample_count": 2023, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1441232, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:22:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T10:22:21+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-01T10:22:21+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-01T10:22:21.167254+09:00", "mtime_epoch": 1782868941.1672542, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166294}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:22:21+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5052.914, "name": "celery", "pid": 267, "rss_mb": 5052.914, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.67, "pushed": 6.67, "read": 6.636}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.254, "count": 199, "max_ms": 49.0, "p95_ms": 35.965}, "gstreamer_push": {"avg_ms": 1.093, "count": 200, "max_ms": 1.81, "p95_ms": 1.147}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.357, "count": 199, "max_ms": 40.114, "p95_ms": 34.979}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.499, "count": 199, "max_ms": 91.143, "p95_ms": 84.502}, "yolo11_640_fire_11class": {"avg_ms": 49.256, "count": 199, "max_ms": 71.381, "p95_ms": 64.234}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.838, "count": 199, "max_ms": 85.963, "p95_ms": 80.929}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.631, "count": 200, "max_ms": 179.663, "p95_ms": 159.525}, "read_interval": {"avg_ms": 150.739, "count": 199, "max_ms": 191.938, "p95_ms": 172.106}, "triton_wait": {"avg_ms": 65.027, "count": 199, "max_ms": 91.155, "p95_ms": 84.645}, "visualize": {"avg_ms": 4.397, "count": 199, "max_ms": 6.043, "p95_ms": 5.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12010}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10895.262, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 486, "h264_decode_error_count": 1422, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:22:21+09:00", "last_error_summary": "[h264 @ 0x55c348ae18c0] error while decoding MB 21 13, bytestream -5", "last_exit_at": "2026-07-01T10:22:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.003, "last_started_at": "2026-07-01T10:19:37+09:00", "pid": 96392, "process": {"exists": false, "pid": 96392}, "restart_count": 533, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 534}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.103, "sample_count": 2023, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1441232, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:22:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T10:22:21+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-01T10:22:21+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-01T10:22:21.167254+09:00", "mtime_epoch": 1782868941.1672542, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166294}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:22:21+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5052.914, "name": "celery", "pid": 267, "rss_mb": 5052.914, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.668, "pushed": 6.668, "read": 6.635}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.254, "count": 199, "max_ms": 49.0, "p95_ms": 35.965}, "gstreamer_push": {"avg_ms": 1.093, "count": 200, "max_ms": 1.81, "p95_ms": 1.147}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.357, "count": 199, "max_ms": 40.114, "p95_ms": 34.979}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.499, "count": 199, "max_ms": 91.143, "p95_ms": 84.502}, "yolo11_640_fire_11class": {"avg_ms": 49.256, "count": 199, "max_ms": 71.381, "p95_ms": 64.234}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.838, "count": 199, "max_ms": 85.963, "p95_ms": 80.929}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.631, "count": 200, "max_ms": 179.663, "p95_ms": 159.525}, "read_interval": {"avg_ms": 150.739, "count": 199, "max_ms": 191.938, "p95_ms": 172.106}, "triton_wait": {"avg_ms": 65.027, "count": 199, "max_ms": 91.155, "p95_ms": 84.645}, "visualize": {"avg_ms": 4.397, "count": 199, "max_ms": 6.043, "p95_ms": 5.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12010}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11000.02, "limit_mb": 8796093022207.996}, "detail": {"pid": 96499, "restart_count": 534, "start_count": 535}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 486, "h264_decode_error_count": 1422, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:22:21+09:00", "last_error_summary": "[h264 @ 0x55c348ae18c0] error while decoding MB 21 13, bytestream -5", "last_exit_at": "2026-07-01T10:22:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:22:23+09:00", "pid": 96499, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.438, "name": "ffmpeg", "pid": 96499, "rss_mb": 136.438, "state": "S (sleeping)", "threads": 106}, "restart_count": 534, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 535}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.183, "sample_count": 1992, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1441232, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:22:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.485, "last_frame_pushed_at": "2026-07-01T10:22:21+09:00", "last_frame_read_age_sec": 4.62, "last_frame_read_at": "2026-07-01T10:22:21+09:00", "mjpeg": {"age_sec": 4.467, "exists": true, "mtime": "2026-07-01T10:22:21.311258+09:00", "mtime_epoch": 1782868941.311258, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166385}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:22:25+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5052.914, "name": "celery", "pid": 267, "rss_mb": 5052.914, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 169, "pushed_frames": 169, "read_frames": 168, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.658, "pushed": 5.658, "read": 5.624}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.119, "count": 169, "max_ms": 47.147, "p95_ms": 34.823}, "gstreamer_push": {"avg_ms": 1.086, "count": 169, "max_ms": 1.219, "p95_ms": 1.146}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.46, "count": 169, "max_ms": 40.114, "p95_ms": 34.979}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.605, "count": 169, "max_ms": 91.143, "p95_ms": 84.502}, "yolo11_640_fire_11class": {"avg_ms": 49.464, "count": 169, "max_ms": 71.381, "p95_ms": 64.234}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.281, "count": 169, "max_ms": 85.963, "p95_ms": 80.929}}, "truncated_count": 0}, "processing_total": {"avg_ms": 138.974, "count": 169, "max_ms": 179.663, "p95_ms": 159.556}, "read_interval": {"avg_ms": 151.079, "count": 168, "max_ms": 191.938, "p95_ms": 172.106}, "triton_wait": {"avg_ms": 65.183, "count": 169, "max_ms": 91.155, "p95_ms": 84.645}, "visualize": {"avg_ms": 4.396, "count": 169, "max_ms": 5.548, "p95_ms": 5.324}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10897.344, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55877d9864c0] error while decoding MB 17 20, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 487, "h264_decode_error_count": 1423, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:25:07+09:00", "last_error_summary": "[h264 @ 0x55877d9864c0] error while decoding MB 17 20, bytestream -10", "last_exit_at": "2026-07-01T10:22:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.499, "last_started_at": "2026-07-01T10:22:23+09:00", "pid": 96499, "process": {"exists": false, "pid": 96499}, "restart_count": 534, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 535}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.108, "sample_count": 1973, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1442304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:25:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T10:25:07+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T10:25:07+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T10:25:07.231620+09:00", "mtime_epoch": 1782869107.2316198, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169086}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:25:07+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5053.43, "name": "celery", "pid": 267, "rss_mb": 5053.43, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.337, "pushed": 6.337, "read": 6.337}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.507, "count": 189, "max_ms": 48.263, "p95_ms": 35.581}, "gstreamer_push": {"avg_ms": 1.05, "count": 190, "max_ms": 1.186, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.522, "count": 189, "max_ms": 41.302, "p95_ms": 35.634}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.744, "count": 189, "max_ms": 89.893, "p95_ms": 85.89}, "yolo11_640_fire_11class": {"avg_ms": 49.377, "count": 189, "max_ms": 68.017, "p95_ms": 63.756}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.234, "count": 189, "max_ms": 84.849, "p95_ms": 80.296}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.759, "count": 190, "max_ms": 206.213, "p95_ms": 182.304}, "read_interval": {"avg_ms": 158.215, "count": 190, "max_ms": 216.763, "p95_ms": 192.516}, "triton_wait": {"avg_ms": 64.836, "count": 189, "max_ms": 89.904, "p95_ms": 85.902}, "visualize": {"avg_ms": 4.719, "count": 189, "max_ms": 7.11, "p95_ms": 6.033}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10896.934, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:25:07+09:00", "last_error_summary": "[h264 @ 0x55877d9864c0] error while decoding MB 17 20, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 487, "h264_decode_error_count": 1423, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:25:07+09:00", "last_error_summary": "[h264 @ 0x55877d9864c0] error while decoding MB 17 20, bytestream -10", "last_exit_at": "2026-07-01T10:25:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.587, "last_started_at": "2026-07-01T10:22:23+09:00", "pid": 96499, "process": {"exists": false, "pid": 96499}, "restart_count": 534, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 535}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.108, "sample_count": 1973, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1442305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:25:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T10:25:07+09:00", "last_frame_read_age_sec": 0.14, "last_frame_read_at": "2026-07-01T10:25:07+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-01T10:25:07.231620+09:00", "mtime_epoch": 1782869107.2316198, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169086}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:25:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5053.43, "name": "celery", "pid": 267, "rss_mb": 5053.43, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.349, "pushed": 6.349, "read": 6.315}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.5, "count": 190, "max_ms": 48.263, "p95_ms": 35.581}, "gstreamer_push": {"avg_ms": 1.05, "count": 190, "max_ms": 1.186, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.519, "count": 190, "max_ms": 41.302, "p95_ms": 35.634}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.737, "count": 190, "max_ms": 89.893, "p95_ms": 85.89}, "yolo11_640_fire_11class": {"avg_ms": 49.378, "count": 190, "max_ms": 68.017, "p95_ms": 63.756}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.223, "count": 190, "max_ms": 84.849, "p95_ms": 80.296}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.787, "count": 190, "max_ms": 206.213, "p95_ms": 182.304}, "read_interval": {"avg_ms": 158.308, "count": 189, "max_ms": 216.763, "p95_ms": 192.516}, "triton_wait": {"avg_ms": 64.828, "count": 190, "max_ms": 89.904, "p95_ms": 85.902}, "visualize": {"avg_ms": 4.72, "count": 190, "max_ms": 7.11, "p95_ms": 6.033}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10896.793, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 487, "h264_decode_error_count": 1423, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:25:07+09:00", "last_error_summary": "[h264 @ 0x55877d9864c0] error while decoding MB 17 20, bytestream -10", "last_exit_at": "2026-07-01T10:25:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.592, "last_started_at": "2026-07-01T10:22:23+09:00", "pid": 96499, "process": {"exists": false, "pid": 96499}, "restart_count": 534, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 535}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.108, "sample_count": 1973, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1442305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:25:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T10:25:07+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-01T10:25:07+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-01T10:25:07.231620+09:00", "mtime_epoch": 1782869107.2316198, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169086}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:25:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5053.43, "name": "celery", "pid": 267, "rss_mb": 5053.43, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.348, "pushed": 6.348, "read": 6.314}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.5, "count": 190, "max_ms": 48.263, "p95_ms": 35.581}, "gstreamer_push": {"avg_ms": 1.05, "count": 190, "max_ms": 1.186, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.519, "count": 190, "max_ms": 41.302, "p95_ms": 35.634}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.737, "count": 190, "max_ms": 89.893, "p95_ms": 85.89}, "yolo11_640_fire_11class": {"avg_ms": 49.378, "count": 190, "max_ms": 68.017, "p95_ms": 63.756}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.223, "count": 190, "max_ms": 84.849, "p95_ms": 80.296}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.787, "count": 190, "max_ms": 206.213, "p95_ms": 182.304}, "read_interval": {"avg_ms": 158.308, "count": 189, "max_ms": 216.763, "p95_ms": 192.516}, "triton_wait": {"avg_ms": 64.828, "count": 190, "max_ms": 89.904, "p95_ms": 85.902}, "visualize": {"avg_ms": 4.72, "count": 190, "max_ms": 7.11, "p95_ms": 6.033}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11002.277, "limit_mb": 8796093022207.996}, "detail": {"pid": 96606, "restart_count": 535, "start_count": 536}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 487, "h264_decode_error_count": 1423, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:25:07+09:00", "last_error_summary": "[h264 @ 0x55877d9864c0] error while decoding MB 17 20, bytestream -10", "last_exit_at": "2026-07-01T10:25:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:25:09+09:00", "pid": 96606, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.281, "name": "ffmpeg", "pid": 96606, "rss_mb": 136.281, "state": "S (sleeping)", "threads": 106}, "restart_count": 535, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 536}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.196, "sample_count": 1944, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1442305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:25:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.18, "last_frame_pushed_at": "2026-07-01T10:25:07+09:00", "last_frame_read_age_sec": 4.316, "last_frame_read_at": "2026-07-01T10:25:07+09:00", "mjpeg": {"age_sec": 4.161, "exists": true, "mtime": "2026-07-01T10:25:07.379624+09:00", "mtime_epoch": 1782869107.3796237, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169222}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:25:11+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5053.43, "name": "celery", "pid": 267, "rss_mb": 5053.43, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.386, "pushed": 5.386, "read": 5.353}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.69, "count": 161, "max_ms": 48.263, "p95_ms": 35.972}, "gstreamer_push": {"avg_ms": 1.042, "count": 161, "max_ms": 1.186, "p95_ms": 1.14}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.544, "count": 161, "max_ms": 41.302, "p95_ms": 35.719}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.062, "count": 161, "max_ms": 89.893, "p95_ms": 85.996}, "yolo11_640_fire_11class": {"avg_ms": 49.263, "count": 161, "max_ms": 68.017, "p95_ms": 63.723}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.323, "count": 161, "max_ms": 84.849, "p95_ms": 80.296}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.182, "count": 161, "max_ms": 206.213, "p95_ms": 182.483}, "read_interval": {"avg_ms": 160.661, "count": 160, "max_ms": 216.763, "p95_ms": 193.219}, "triton_wait": {"avg_ms": 65.168, "count": 161, "max_ms": 89.904, "p95_ms": 86.008}, "visualize": {"avg_ms": 4.803, "count": 161, "max_ms": 7.11, "p95_ms": 6.051}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12028}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10899.34, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561ff7bb7700] error while decoding MB 44 30, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 488, "h264_decode_error_count": 1424, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:27:53+09:00", "last_error_summary": "[h264 @ 0x561ff7bb7700] error while decoding MB 44 30, bytestream -5", "last_exit_at": "2026-07-01T10:25:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.62, "last_started_at": "2026-07-01T10:25:09+09:00", "pid": 96606, "process": {"exists": false, "pid": 96606}, "restart_count": 535, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 536}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.176, "sample_count": 1972, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1443387, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:27:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T10:27:53+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T10:27:53+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T10:27:53.115582+09:00", "mtime_epoch": 1782869273.1155818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:27:53+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5053.945, "name": "celery", "pid": 267, "rss_mb": 5053.945, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.753, "pushed": 6.753, "read": 6.753}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.138, "count": 202, "max_ms": 39.307, "p95_ms": 35.23}, "gstreamer_push": {"avg_ms": 1.079, "count": 202, "max_ms": 1.543, "p95_ms": 1.156}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.893, "count": 202, "max_ms": 40.253, "p95_ms": 35.292}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.209, "count": 202, "max_ms": 87.543, "p95_ms": 83.147}, "yolo11_640_fire_11class": {"avg_ms": 47.058, "count": 202, "max_ms": 66.565, "p95_ms": 61.781}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.952, "count": 202, "max_ms": 82.991, "p95_ms": 79.621}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.27, "count": 202, "max_ms": 182.812, "p95_ms": 158.618}, "read_interval": {"avg_ms": 148.581, "count": 202, "max_ms": 193.399, "p95_ms": 168.19}, "triton_wait": {"avg_ms": 63.389, "count": 202, "max_ms": 87.555, "p95_ms": 83.157}, "visualize": {"avg_ms": 4.451, "count": 202, "max_ms": 6.956, "p95_ms": 5.44}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12028}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10898.602, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:27:53+09:00", "last_error_summary": "[h264 @ 0x561ff7bb7700] error while decoding MB 44 30, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 488, "h264_decode_error_count": 1424, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:27:53+09:00", "last_error_summary": "[h264 @ 0x561ff7bb7700] error while decoding MB 44 30, bytestream -5", "last_exit_at": "2026-07-01T10:27:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.706, "last_started_at": "2026-07-01T10:25:09+09:00", "pid": 96606, "process": {"exists": false, "pid": 96606}, "restart_count": 535, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 536}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.176, "sample_count": 1972, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1443388, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:27:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T10:27:53+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-01T10:27:53+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T10:27:53.115582+09:00", "mtime_epoch": 1782869273.1155818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:27:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5053.945, "name": "celery", "pid": 267, "rss_mb": 5053.945, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.767, "pushed": 6.767, "read": 6.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.138, "count": 202, "max_ms": 39.307, "p95_ms": 35.23}, "gstreamer_push": {"avg_ms": 1.079, "count": 203, "max_ms": 1.543, "p95_ms": 1.156}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.894, "count": 202, "max_ms": 40.253, "p95_ms": 35.292}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.09, "count": 202, "max_ms": 87.543, "p95_ms": 82.736}, "yolo11_640_fire_11class": {"avg_ms": 46.954, "count": 202, "max_ms": 66.565, "p95_ms": 59.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.834, "count": 202, "max_ms": 82.991, "p95_ms": 79.621}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.236, "count": 203, "max_ms": 182.812, "p95_ms": 158.618}, "read_interval": {"avg_ms": 148.581, "count": 202, "max_ms": 193.399, "p95_ms": 168.19}, "triton_wait": {"avg_ms": 63.271, "count": 202, "max_ms": 87.555, "p95_ms": 82.752}, "visualize": {"avg_ms": 4.45, "count": 202, "max_ms": 6.956, "p95_ms": 5.44}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12028}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10898.578, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 488, "h264_decode_error_count": 1424, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:27:53+09:00", "last_error_summary": "[h264 @ 0x561ff7bb7700] error while decoding MB 44 30, bytestream -5", "last_exit_at": "2026-07-01T10:27:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.711, "last_started_at": "2026-07-01T10:25:09+09:00", "pid": 96606, "process": {"exists": false, "pid": 96606}, "restart_count": 535, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 536}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.176, "sample_count": 1972, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1443388, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:27:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T10:27:53+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T10:27:53+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T10:27:53.115582+09:00", "mtime_epoch": 1782869273.1155818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:27:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5053.945, "name": "celery", "pid": 267, "rss_mb": 5053.945, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.736, "pushed": 6.736, "read": 6.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.138, "count": 202, "max_ms": 39.307, "p95_ms": 35.23}, "gstreamer_push": {"avg_ms": 1.079, "count": 202, "max_ms": 1.543, "p95_ms": 1.156}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.894, "count": 202, "max_ms": 40.253, "p95_ms": 35.292}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.09, "count": 202, "max_ms": 87.543, "p95_ms": 82.736}, "yolo11_640_fire_11class": {"avg_ms": 46.954, "count": 202, "max_ms": 66.565, "p95_ms": 59.886}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.834, "count": 202, "max_ms": 82.991, "p95_ms": 79.621}}, "truncated_count": 0}, "processing_total": {"avg_ms": 137.156, "count": 202, "max_ms": 182.812, "p95_ms": 158.618}, "read_interval": {"avg_ms": 148.581, "count": 202, "max_ms": 193.399, "p95_ms": 168.19}, "triton_wait": {"avg_ms": 63.271, "count": 202, "max_ms": 87.555, "p95_ms": 82.752}, "visualize": {"avg_ms": 4.45, "count": 202, "max_ms": 6.956, "p95_ms": 5.44}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12028}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11004.227, "limit_mb": 8796093022207.996}, "detail": {"pid": 96713, "restart_count": 536, "start_count": 537}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 488, "h264_decode_error_count": 1424, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:27:53+09:00", "last_error_summary": "[h264 @ 0x561ff7bb7700] error while decoding MB 44 30, bytestream -5", "last_exit_at": "2026-07-01T10:27:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:27:55+09:00", "pid": 96713, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.18, "name": "ffmpeg", "pid": 96713, "rss_mb": 137.18, "state": "S (sleeping)", "threads": 106}, "restart_count": 536, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 537}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.262, "sample_count": 1943, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1443388, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:27:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.473, "last_frame_pushed_at": "2026-07-01T10:27:53+09:00", "last_frame_read_age_sec": 4.605, "last_frame_read_at": "2026-07-01T10:27:53+09:00", "mjpeg": {"age_sec": 4.454, "exists": true, "mtime": "2026-07-01T10:27:53.259585+09:00", "mtime_epoch": 1782869273.259585, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170461}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:27:57+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5053.945, "name": "celery", "pid": 267, "rss_mb": 5053.945, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 173, "pushed_frames": 173, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.767, "pushed": 5.767, "read": 5.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.147, "count": 173, "max_ms": 39.307, "p95_ms": 35.23}, "gstreamer_push": {"avg_ms": 1.077, "count": 173, "max_ms": 1.543, "p95_ms": 1.143}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.934, "count": 173, "max_ms": 40.253, "p95_ms": 35.424}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.574, "count": 173, "max_ms": 87.543, "p95_ms": 83.147}, "yolo11_640_fire_11class": {"avg_ms": 46.587, "count": 173, "max_ms": 66.565, "p95_ms": 58.582}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.598, "count": 173, "max_ms": 82.991, "p95_ms": 79.621}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.227, "count": 173, "max_ms": 182.812, "p95_ms": 158.618}, "read_interval": {"avg_ms": 147.713, "count": 173, "max_ms": 193.399, "p95_ms": 169.481}, "triton_wait": {"avg_ms": 62.741, "count": 173, "max_ms": 87.555, "p95_ms": 83.157}, "visualize": {"avg_ms": 4.421, "count": 173, "max_ms": 6.809, "p95_ms": 5.366}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10900.359, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55cf5b9077c0] error while decoding MB 14 24, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 489, "h264_decode_error_count": 1425, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:30:06+09:00", "last_error_summary": "[h264 @ 0x55cf5b9077c0] error while decoding MB 14 24, bytestream -10", "last_exit_at": "2026-07-01T10:27:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.26, "last_started_at": "2026-07-01T10:27:55+09:00", "pid": 96713, "process": {"exists": false, "pid": 96713}, "restart_count": 536, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 537}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.801, "sample_count": 1829, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444131, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:30:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T10:30:06+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T10:30:06+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T10:30:06.930563+09:00", "mtime_epoch": 1782869406.9305634, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172504}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:30:06+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5054.461, "name": "celery", "pid": 267, "rss_mb": 5054.461, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 174, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.82, "pushed": 5.82, "read": 5.82}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.569, "count": 174, "max_ms": 44.643, "p95_ms": 39.047}, "gstreamer_push": {"avg_ms": 1.0, "count": 174, "max_ms": 1.441, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.37, "count": 174, "max_ms": 45.034, "p95_ms": 34.996}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.551, "count": 174, "max_ms": 90.132, "p95_ms": 82.51}, "yolo11_640_fire_11class": {"avg_ms": 47.542, "count": 174, "max_ms": 64.936, "p95_ms": 61.209}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.941, "count": 174, "max_ms": 83.875, "p95_ms": 79.365}}, "truncated_count": 0}, "processing_total": {"avg_ms": 161.788, "count": 174, "max_ms": 198.013, "p95_ms": 182.456}, "read_interval": {"avg_ms": 172.423, "count": 174, "max_ms": 208.346, "p95_ms": 193.562}, "triton_wait": {"avg_ms": 63.603, "count": 174, "max_ms": 90.144, "p95_ms": 82.522}, "visualize": {"avg_ms": 5.302, "count": 174, "max_ms": 6.549, "p95_ms": 6.208}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10899.52, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:30:06+09:00", "last_error_summary": "[h264 @ 0x55cf5b9077c0] error while decoding MB 14 24, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 489, "h264_decode_error_count": 1425, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:30:06+09:00", "last_error_summary": "[h264 @ 0x55cf5b9077c0] error while decoding MB 14 24, bytestream -10", "last_exit_at": "2026-07-01T10:30:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.379, "last_started_at": "2026-07-01T10:27:55+09:00", "pid": 96713, "process": {"exists": false, "pid": 96713}, "restart_count": 536, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 537}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.801, "sample_count": 1829, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:30:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T10:30:07+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T10:30:06+09:00", "mjpeg": {"age_sec": 0.161, "exists": true, "mtime": "2026-07-01T10:30:06.930563+09:00", "mtime_epoch": 1782869406.9305634, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172504}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:30:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5054.461, "name": "celery", "pid": 267, "rss_mb": 5054.461, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.831, "pushed": 5.831, "read": 5.798}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.566, "count": 174, "max_ms": 44.643, "p95_ms": 39.047}, "gstreamer_push": {"avg_ms": 1.0, "count": 174, "max_ms": 1.441, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.361, "count": 174, "max_ms": 45.034, "p95_ms": 34.996}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.565, "count": 174, "max_ms": 90.132, "p95_ms": 82.51}, "yolo11_640_fire_11class": {"avg_ms": 47.553, "count": 174, "max_ms": 64.936, "p95_ms": 61.209}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.971, "count": 174, "max_ms": 83.875, "p95_ms": 79.365}}, "truncated_count": 0}, "processing_total": {"avg_ms": 161.815, "count": 174, "max_ms": 198.013, "p95_ms": 182.456}, "read_interval": {"avg_ms": 172.452, "count": 173, "max_ms": 208.346, "p95_ms": 193.562}, "triton_wait": {"avg_ms": 63.618, "count": 174, "max_ms": 90.144, "p95_ms": 82.522}, "visualize": {"avg_ms": 5.301, "count": 174, "max_ms": 6.549, "p95_ms": 6.208}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.84, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10899.562, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 489, "h264_decode_error_count": 1425, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:30:06+09:00", "last_error_summary": "[h264 @ 0x55cf5b9077c0] error while decoding MB 14 24, bytestream -10", "last_exit_at": "2026-07-01T10:30:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.382, "last_started_at": "2026-07-01T10:27:55+09:00", "pid": 96713, "process": {"exists": false, "pid": 96713}, "restart_count": 536, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 537}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.801, "sample_count": 1829, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:30:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T10:30:07+09:00", "last_frame_read_age_sec": 0.171, "last_frame_read_at": "2026-07-01T10:30:06+09:00", "mjpeg": {"age_sec": 0.165, "exists": true, "mtime": "2026-07-01T10:30:06.930563+09:00", "mtime_epoch": 1782869406.9305634, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172504}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:30:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5054.461, "name": "celery", "pid": 267, "rss_mb": 5054.461, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.83, "pushed": 5.83, "read": 5.797}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.566, "count": 174, "max_ms": 44.643, "p95_ms": 39.047}, "gstreamer_push": {"avg_ms": 1.0, "count": 174, "max_ms": 1.441, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.361, "count": 174, "max_ms": 45.034, "p95_ms": 34.996}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.565, "count": 174, "max_ms": 90.132, "p95_ms": 82.51}, "yolo11_640_fire_11class": {"avg_ms": 47.553, "count": 174, "max_ms": 64.936, "p95_ms": 61.209}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.971, "count": 174, "max_ms": 83.875, "p95_ms": 79.365}}, "truncated_count": 0}, "processing_total": {"avg_ms": 161.815, "count": 174, "max_ms": 198.013, "p95_ms": 182.456}, "read_interval": {"avg_ms": 172.452, "count": 173, "max_ms": 208.346, "p95_ms": 193.562}, "triton_wait": {"avg_ms": 63.618, "count": 174, "max_ms": 90.144, "p95_ms": 82.522}, "visualize": {"avg_ms": 5.301, "count": 174, "max_ms": 6.549, "p95_ms": 6.208}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.844, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11004.867, "limit_mb": 8796093022207.996}, "detail": {"pid": 96820, "restart_count": 537, "start_count": 538}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 489, "h264_decode_error_count": 1425, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:30:06+09:00", "last_error_summary": "[h264 @ 0x55cf5b9077c0] error while decoding MB 14 24, bytestream -10", "last_exit_at": "2026-07-01T10:30:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:30:09+09:00", "pid": 96820, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.648, "name": "ffmpeg", "pid": 96820, "rss_mb": 136.648, "state": "S (sleeping)", "threads": 106}, "restart_count": 537, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 538}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.801, "sample_count": 1826, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1444132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:30:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.62, "last_frame_pushed_at": "2026-07-01T10:30:07+09:00", "last_frame_read_age_sec": 4.782, "last_frame_read_at": "2026-07-01T10:30:06+09:00", "mjpeg": {"age_sec": 4.604, "exists": true, "mtime": "2026-07-01T10:30:07.102567+09:00", "mtime_epoch": 1782869407.1025672, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172547}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:30:11+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5054.461, "name": "celery", "pid": 267, "rss_mb": 5054.461, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.977, "pushed": 4.977, "read": 4.943}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.466, "count": 148, "max_ms": 44.643, "p95_ms": 39.44}, "gstreamer_push": {"avg_ms": 1.005, "count": 149, "max_ms": 1.205, "p95_ms": 1.151}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.413, "count": 148, "max_ms": 45.034, "p95_ms": 34.996}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.617, "count": 148, "max_ms": 90.132, "p95_ms": 82.51}, "yolo11_640_fire_11class": {"avg_ms": 47.483, "count": 148, "max_ms": 64.936, "p95_ms": 61.209}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.989, "count": 148, "max_ms": 83.875, "p95_ms": 79.365}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.519, "count": 149, "max_ms": 198.013, "p95_ms": 182.456}, "read_interval": {"avg_ms": 171.145, "count": 148, "max_ms": 208.346, "p95_ms": 193.562}, "triton_wait": {"avg_ms": 63.676, "count": 148, "max_ms": 90.144, "p95_ms": 82.522}, "visualize": {"avg_ms": 5.246, "count": 148, "max_ms": 6.549, "p95_ms": 6.13}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10842.398, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555d44e09ec0] error while decoding MB 11 57, bytestream -15 [in#0/rtsp @ 0x555d44d87e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d44d87e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 212, "h264_decode_error_count": 218, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:32:04+09:00", "last_error_summary": "[h264 @ 0x555d44e09ec0] error while decoding MB 11 57, bytestream -15 [in#0/rtsp @ 0x555d44d87e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d44d87e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:19:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 783.933, "last_started_at": "2026-07-01T10:19:00+09:00", "pid": 96240, "process": {"exists": false, "pid": 96240}, "restart_count": 211, "restart_count_per_hour": 6, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 212}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.258, "sample_count": 2617, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1643330, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:32:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.086, "last_frame_pushed_at": "2026-07-01T10:32:04+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-01T10:32:04+09:00", "mjpeg": {"age_sec": 0.069, "exists": true, "mtime": "2026-07-01T10:32:04.693067+09:00", "mtime_epoch": 1782869524.693067, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224885}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:32:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1839.672, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.726, "pushed": 0.726, "read": 0.781}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.15, "count": 13, "max_ms": 27.251, "p95_ms": 27.251}, "gstreamer_push": {"avg_ms": 1.143, "count": 13, "max_ms": 1.232, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.285, "count": 13, "max_ms": 55.799, "p95_ms": 55.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.909, "count": 13, "max_ms": 83.289, "p95_ms": 83.289}, "yolo11_640_fire_11class": {"avg_ms": 47.926, "count": 13, "max_ms": 60.905, "p95_ms": 60.905}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.636, "count": 13, "max_ms": 79.697, "p95_ms": 79.697}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.894, "count": 13, "max_ms": 111.204, "p95_ms": 111.204}, "read_interval": {"avg_ms": 2424.554, "count": 14, "max_ms": 16653.677, "p95_ms": 16653.677}, "triton_wait": {"avg_ms": 56.922, "count": 13, "max_ms": 83.301, "p95_ms": 83.301}, "visualize": {"avg_ms": 3.317, "count": 13, "max_ms": 3.537, "p95_ms": 3.537}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.919, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10841.996, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:32:04+09:00", "last_error_summary": "[h264 @ 0x555d44e09ec0] error while decoding MB 11 57, bytestream -15 [in#0/rtsp @ 0x555d44d87e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d44d87e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 212, "h264_decode_error_count": 218, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:32:04+09:00", "last_error_summary": "[h264 @ 0x555d44e09ec0] error while decoding MB 11 57, bytestream -15 [in#0/rtsp @ 0x555d44d87e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d44d87e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:32:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 783.956, "last_started_at": "2026-07-01T10:19:00+09:00", "pid": 96240, "process": {"exists": false, "pid": 96240}, "restart_count": 211, "restart_count_per_hour": 6, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 212}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.258, "sample_count": 2618, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1643331, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:32:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T10:32:04+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-01T10:32:04+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-01T10:32:04.693067+09:00", "mtime_epoch": 1782869524.693067, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224885}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:32:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1839.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.78, "pushed": 0.78, "read": 0.78}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.363, "count": 14, "max_ms": 27.251, "p95_ms": 27.251}, "gstreamer_push": {"avg_ms": 1.149, "count": 14, "max_ms": 1.232, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.18, "count": 14, "max_ms": 55.799, "p95_ms": 55.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.607, "count": 14, "max_ms": 83.289, "p95_ms": 83.289}, "yolo11_640_fire_11class": {"avg_ms": 48.158, "count": 14, "max_ms": 60.905, "p95_ms": 60.905}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.552, "count": 14, "max_ms": 79.697, "p95_ms": 79.697}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.954, "count": 14, "max_ms": 111.204, "p95_ms": 111.204}, "read_interval": {"avg_ms": 2424.554, "count": 14, "max_ms": 16653.677, "p95_ms": 16653.677}, "triton_wait": {"avg_ms": 56.62, "count": 14, "max_ms": 83.301, "p95_ms": 83.301}, "visualize": {"avg_ms": 3.447, "count": 14, "max_ms": 5.126, "p95_ms": 5.126}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.941, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10841.965, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 212, "h264_decode_error_count": 218, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:32:04+09:00", "last_error_summary": "[h264 @ 0x555d44e09ec0] error while decoding MB 11 57, bytestream -15 [in#0/rtsp @ 0x555d44d87e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d44d87e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:32:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 783.964, "last_started_at": "2026-07-01T10:19:00+09:00", "pid": 96240, "process": {"exists": false, "pid": 96240}, "restart_count": 211, "restart_count_per_hour": 6, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 212}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.258, "sample_count": 2618, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1643331, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:32:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T10:32:04+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T10:32:04+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-01T10:32:04.693067+09:00", "mtime_epoch": 1782869524.693067, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224885}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:32:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1839.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.78, "pushed": 0.78, "read": 0.78}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.363, "count": 14, "max_ms": 27.251, "p95_ms": 27.251}, "gstreamer_push": {"avg_ms": 1.149, "count": 14, "max_ms": 1.232, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.18, "count": 14, "max_ms": 55.799, "p95_ms": 55.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.607, "count": 14, "max_ms": 83.289, "p95_ms": 83.289}, "yolo11_640_fire_11class": {"avg_ms": 48.158, "count": 14, "max_ms": 60.905, "p95_ms": 60.905}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.552, "count": 14, "max_ms": 79.697, "p95_ms": 79.697}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.954, "count": 14, "max_ms": 111.204, "p95_ms": 111.204}, "read_interval": {"avg_ms": 2424.554, "count": 14, "max_ms": 16653.677, "p95_ms": 16653.677}, "triton_wait": {"avg_ms": 56.62, "count": 14, "max_ms": 83.301, "p95_ms": 83.301}, "visualize": {"avg_ms": 3.447, "count": 14, "max_ms": 5.126, "p95_ms": 5.126}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.949, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10981.707, "limit_mb": 8796093022207.996}, "detail": {"pid": 96927, "restart_count": 212, "start_count": 213}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 212, "h264_decode_error_count": 218, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:32:04+09:00", "last_error_summary": "[h264 @ 0x555d44e09ec0] error while decoding MB 11 57, bytestream -15 [in#0/rtsp @ 0x555d44d87e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d44d87e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:32:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:32:07+09:00", "pid": 96927, "process": {"exists": true, "fd_count": 5, "hwm_mb": 168.574, "name": "ffmpeg", "pid": 96927, "rss_mb": 168.574, "state": "R (running)", "threads": 106}, "restart_count": 212, "restart_count_per_hour": 7, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 213}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.184, "sample_count": 2572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1643331, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:32:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.698, "last_frame_pushed_at": "2026-07-01T10:32:04+09:00", "last_frame_read_age_sec": 4.787, "last_frame_read_at": "2026-07-01T10:32:04+09:00", "mjpeg": {"age_sec": 4.68, "exists": true, "mtime": "2026-07-01T10:32:04.797069+09:00", "mtime_epoch": 1782869524.797069, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221568}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:32:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1839.672, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.619, "pushed": 0.619, "read": 0.619}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 21.363, "count": 14, "max_ms": 27.251, "p95_ms": 27.251}, "gstreamer_push": {"avg_ms": 1.149, "count": 14, "max_ms": 1.232, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.18, "count": 14, "max_ms": 55.799, "p95_ms": 55.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.607, "count": 14, "max_ms": 83.289, "p95_ms": 83.289}, "yolo11_640_fire_11class": {"avg_ms": 48.158, "count": 14, "max_ms": 60.905, "p95_ms": 60.905}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.552, "count": 14, "max_ms": 79.697, "p95_ms": 79.697}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.954, "count": 14, "max_ms": 111.204, "p95_ms": 111.204}, "read_interval": {"avg_ms": 2424.554, "count": 14, "max_ms": 16653.677, "p95_ms": 16653.677}, "triton_wait": {"avg_ms": 56.62, "count": 14, "max_ms": 83.301, "p95_ms": 83.301}, "visualize": {"avg_ms": 3.447, "count": 14, "max_ms": 5.126, "p95_ms": 5.126}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.634, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12043}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10900.254, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a7608db940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7608db940] error while decoding MB 0 38, bytestream 1320 [h264 @ 0x55a75fc0d4c0] left block unavailable for requested intra mode [h264 @ 0x55a75fc0d4c0] error while decoding MB 0 2, bytestream 7250 [h264 @ 0x55a7605a00c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7605a00c0] error while decoding MB 0 36, bytestream 1018 [h264 @ 0x55a75fbf9340] error while decoding MB 25 29, bytestre", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 490, "h264_decode_error_count": 1429, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:32:59+09:00", "last_error_summary": "[h264 @ 0x55a7608db940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7608db940] error while decoding MB 0 38, bytestream 1320 [h264 @ 0x55a75fc0d4c0] left block unavailable for requested intra mode [h264 @ 0x55a75fc0d4c0] error while decoding MB 0 2, bytestream 7250 [h264 @ 0x55a7605a00c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7605a00c0] error while decoding MB 0 36, bytestream 1018 [h264 @ 0x55a75fbf9340] error while decoding MB 25 29, bytestre", "last_exit_at": "2026-07-01T10:30:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 169.863, "last_started_at": "2026-07-01T10:30:09+09:00", "pid": 96820, "process": {"exists": false, "pid": 96820}, "restart_count": 537, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 538}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.042, "sample_count": 1782, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1445180, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:32:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T10:32:59+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T10:32:59+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T10:32:59.526201+09:00", "mtime_epoch": 1782869579.5262012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176596}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:32:59+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5054.977, "name": "celery", "pid": 267, "rss_mb": 5054.977, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.158, "pushed": 6.158, "read": 6.158}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.164, "count": 184, "max_ms": 39.097, "p95_ms": 37.34}, "gstreamer_push": {"avg_ms": 1.027, "count": 184, "max_ms": 1.33, "p95_ms": 1.124}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.137, "count": 184, "max_ms": 44.111, "p95_ms": 36.309}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.078, "count": 184, "max_ms": 88.135, "p95_ms": 83.718}, "yolo11_640_fire_11class": {"avg_ms": 46.927, "count": 184, "max_ms": 66.085, "p95_ms": 62.665}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.333, "count": 184, "max_ms": 84.943, "p95_ms": 80.852}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.749, "count": 184, "max_ms": 189.912, "p95_ms": 180.775}, "read_interval": {"avg_ms": 162.777, "count": 184, "max_ms": 199.854, "p95_ms": 188.982}, "triton_wait": {"avg_ms": 64.361, "count": 184, "max_ms": 88.147, "p95_ms": 83.73}, "visualize": {"avg_ms": 4.925, "count": 184, "max_ms": 6.362, "p95_ms": 5.657}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12043}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10900.004, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:32:59+09:00", "last_error_summary": "[h264 @ 0x55a7608db940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7608db940] error while decoding MB 0 38, bytestream 1320 [h264 @ 0x55a75fc0d4c0] left block unavailable for requested intra mode [h264 @ 0x55a75fc0d4c0] error while decoding MB 0 2, bytestream 7250 [h264 @ 0x55a7605a00c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7605a00c0] error while decoding MB 0 36, bytestream 1018 [h264 @ 0x55a75fbf9340] error while decoding MB 25 29, bytestre", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 490, "h264_decode_error_count": 1429, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:32:59+09:00", "last_error_summary": "[h264 @ 0x55a7608db940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7608db940] error while decoding MB 0 38, bytestream 1320 [h264 @ 0x55a75fc0d4c0] left block unavailable for requested intra mode [h264 @ 0x55a75fc0d4c0] error while decoding MB 0 2, bytestream 7250 [h264 @ 0x55a7605a00c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7605a00c0] error while decoding MB 0 36, bytestream 1018 [h264 @ 0x55a75fbf9340] error while decoding MB 25 29, bytestre", "last_exit_at": "2026-07-01T10:32:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 169.971, "last_started_at": "2026-07-01T10:30:09+09:00", "pid": 96820, "process": {"exists": false, "pid": 96820}, "restart_count": 537, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 538}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.042, "sample_count": 1782, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1445181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:32:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T10:32:59+09:00", "last_frame_read_age_sec": 0.164, "last_frame_read_at": "2026-07-01T10:32:59+09:00", "mjpeg": {"age_sec": 0.151, "exists": true, "mtime": "2026-07-01T10:32:59.526201+09:00", "mtime_epoch": 1782869579.5262012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176596}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:32:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5054.977, "name": "celery", "pid": 267, "rss_mb": 5054.977, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 185, "pushed_frames": 185, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.169, "pushed": 6.169, "read": 6.135}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.2, "count": 184, "max_ms": 39.097, "p95_ms": 37.34}, "gstreamer_push": {"avg_ms": 1.027, "count": 185, "max_ms": 1.33, "p95_ms": 1.124}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.147, "count": 184, "max_ms": 44.111, "p95_ms": 36.309}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.095, "count": 184, "max_ms": 88.135, "p95_ms": 83.718}, "yolo11_640_fire_11class": {"avg_ms": 46.925, "count": 184, "max_ms": 66.085, "p95_ms": 62.665}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.344, "count": 184, "max_ms": 84.943, "p95_ms": 80.852}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.781, "count": 185, "max_ms": 189.912, "p95_ms": 180.775}, "read_interval": {"avg_ms": 162.777, "count": 184, "max_ms": 199.854, "p95_ms": 188.982}, "triton_wait": {"avg_ms": 64.379, "count": 184, "max_ms": 88.147, "p95_ms": 83.73}, "visualize": {"avg_ms": 4.932, "count": 184, "max_ms": 6.362, "p95_ms": 5.657}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12043}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10899.996, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 490, "h264_decode_error_count": 1429, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:32:59+09:00", "last_error_summary": "[h264 @ 0x55a7608db940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7608db940] error while decoding MB 0 38, bytestream 1320 [h264 @ 0x55a75fc0d4c0] left block unavailable for requested intra mode [h264 @ 0x55a75fc0d4c0] error while decoding MB 0 2, bytestream 7250 [h264 @ 0x55a7605a00c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7605a00c0] error while decoding MB 0 36, bytestream 1018 [h264 @ 0x55a75fbf9340] error while decoding MB 25 29, bytestre", "last_exit_at": "2026-07-01T10:32:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 169.975, "last_started_at": "2026-07-01T10:30:09+09:00", "pid": 96820, "process": {"exists": false, "pid": 96820}, "restart_count": 537, "restart_count_per_hour": 23, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 538}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 181.042, "sample_count": 1782, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1445181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:32:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T10:32:59+09:00", "last_frame_read_age_sec": 0.168, "last_frame_read_at": "2026-07-01T10:32:59+09:00", "mjpeg": {"age_sec": 0.154, "exists": true, "mtime": "2026-07-01T10:32:59.526201+09:00", "mtime_epoch": 1782869579.5262012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176596}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:32:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5054.977, "name": "celery", "pid": 267, "rss_mb": 5054.977, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 185, "pushed_frames": 185, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.168, "pushed": 6.168, "read": 6.135}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.2, "count": 184, "max_ms": 39.097, "p95_ms": 37.34}, "gstreamer_push": {"avg_ms": 1.027, "count": 185, "max_ms": 1.33, "p95_ms": 1.124}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.147, "count": 184, "max_ms": 44.111, "p95_ms": 36.309}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.095, "count": 184, "max_ms": 88.135, "p95_ms": 83.718}, "yolo11_640_fire_11class": {"avg_ms": 46.925, "count": 184, "max_ms": 66.085, "p95_ms": 62.665}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.344, "count": 184, "max_ms": 84.943, "p95_ms": 80.852}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.781, "count": 185, "max_ms": 189.912, "p95_ms": 180.775}, "read_interval": {"avg_ms": 162.777, "count": 184, "max_ms": 199.854, "p95_ms": 188.982}, "triton_wait": {"avg_ms": 64.379, "count": 184, "max_ms": 88.147, "p95_ms": 83.73}, "visualize": {"avg_ms": 4.932, "count": 184, "max_ms": 6.362, "p95_ms": 5.657}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12043}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11001.891, "limit_mb": 8796093022207.996}, "detail": {"pid": 97079, "restart_count": 538, "start_count": 539}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 490, "h264_decode_error_count": 1429, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:32:59+09:00", "last_error_summary": "[h264 @ 0x55a7608db940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7608db940] error while decoding MB 0 38, bytestream 1320 [h264 @ 0x55a75fc0d4c0] left block unavailable for requested intra mode [h264 @ 0x55a75fc0d4c0] error while decoding MB 0 2, bytestream 7250 [h264 @ 0x55a7605a00c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a7605a00c0] error while decoding MB 0 36, bytestream 1018 [h264 @ 0x55a75fbf9340] error while decoding MB 25 29, bytestre", "last_exit_at": "2026-07-01T10:33:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:33:02+09:00", "pid": 97079, "process": {"exists": true, "fd_count": 5, "hwm_mb": 134.086, "name": "ffmpeg", "pid": 97079, "rss_mb": 134.086, "state": "S (sleeping)", "threads": 106}, "restart_count": 538, "restart_count_per_hour": 24, "rtsp_timeout_count": 406, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 539}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.984, "sample_count": 1754, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1445181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:32:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.928, "last_frame_pushed_at": "2026-07-01T10:32:59+09:00", "last_frame_read_age_sec": 5.088, "last_frame_read_at": "2026-07-01T10:32:59+09:00", "mjpeg": {"age_sec": 4.91, "exists": true, "mtime": "2026-07-01T10:32:59.690205+09:00", "mtime_epoch": 1782869579.6902046, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177025}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:33:04+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5054.977, "name": "celery", "pid": 267, "rss_mb": 5054.977, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 154, "pushed_frames": 154, "read_frames": 153, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.154, "pushed": 5.154, "read": 5.121}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.207, "count": 154, "max_ms": 39.097, "p95_ms": 37.337}, "gstreamer_push": {"avg_ms": 1.028, "count": 154, "max_ms": 1.33, "p95_ms": 1.127}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.653, "count": 154, "max_ms": 44.111, "p95_ms": 37.269}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.875, "count": 154, "max_ms": 88.135, "p95_ms": 83.718}, "yolo11_640_fire_11class": {"avg_ms": 47.485, "count": 154, "max_ms": 66.085, "p95_ms": 62.665}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.776, "count": 154, "max_ms": 84.943, "p95_ms": 80.852}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.036, "count": 154, "max_ms": 189.912, "p95_ms": 180.775}, "read_interval": {"avg_ms": 162.997, "count": 153, "max_ms": 199.854, "p95_ms": 188.982}, "triton_wait": {"avg_ms": 65.073, "count": 154, "max_ms": 88.147, "p95_ms": 83.73}, "visualize": {"avg_ms": 4.901, "count": 154, "max_ms": 6.339, "p95_ms": 5.69}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12051}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10902.746, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ff7cbaf640] left block unavailable for requested intra mode [h264 @ 0x55ff7cbaf640] error while decoding MB 0 16, bytestream 3827 [h264 @ 0x55ff7cc0a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cc0a100] error while decoding MB 0 35, bytestream 2262 [h264 @ 0x55ff7cbaf640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cbaf640] error while decoding MB 0 25, bytestream 1604 [h264 @ 0x55ff7ccb8640] error while decoding MB 17 26, bytestr", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 491, "h264_decode_error_count": 1433, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:36:14+09:00", "last_error_summary": "[h264 @ 0x55ff7cbaf640] left block unavailable for requested intra mode [h264 @ 0x55ff7cbaf640] error while decoding MB 0 16, bytestream 3827 [h264 @ 0x55ff7cc0a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cc0a100] error while decoding MB 0 35, bytestream 2262 [h264 @ 0x55ff7cbaf640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cbaf640] error while decoding MB 0 25, bytestream 1604 [h264 @ 0x55ff7ccb8640] error while decoding MB 17 26, bytestr", "last_exit_at": "2026-07-01T10:33:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.514, "last_started_at": "2026-07-01T10:33:02+09:00", "pid": 97079, "process": {"exists": false, "pid": 97079}, "restart_count": 538, "restart_count_per_hour": 22, "rtsp_timeout_count": 408, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 539}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.667, "sample_count": 1721, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446203, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:36:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T10:36:14+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T10:36:14+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T10:36:14.066099+09:00", "mtime_epoch": 1782869774.0660987, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169725}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:36:14+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5055.492, "name": "celery", "pid": 267, "rss_mb": 5055.492, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 69, "pushed_frames": 69, "read_frames": 69, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.312, "pushed": 2.312, "read": 2.312}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.416, "count": 69, "max_ms": 38.638, "p95_ms": 35.477}, "gstreamer_push": {"avg_ms": 1.018, "count": 69, "max_ms": 1.149, "p95_ms": 1.119}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.03, "count": 69, "max_ms": 44.104, "p95_ms": 40.437}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.15, "count": 69, "max_ms": 94.748, "p95_ms": 84.003}, "yolo11_640_fire_11class": {"avg_ms": 49.367, "count": 69, "max_ms": 70.019, "p95_ms": 62.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.731, "count": 69, "max_ms": 83.544, "p95_ms": 78.464}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.649, "count": 69, "max_ms": 189.253, "p95_ms": 177.243}, "read_interval": {"avg_ms": 434.045, "count": 69, "max_ms": 18570.815, "p95_ms": 189.981}, "triton_wait": {"avg_ms": 63.512, "count": 69, "max_ms": 94.761, "p95_ms": 84.015}, "visualize": {"avg_ms": 5.116, "count": 69, "max_ms": 6.011, "p95_ms": 5.626}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.847, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12051}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10902.434, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:36:14+09:00", "last_error_summary": "[h264 @ 0x55ff7cbaf640] left block unavailable for requested intra mode [h264 @ 0x55ff7cbaf640] error while decoding MB 0 16, bytestream 3827 [h264 @ 0x55ff7cc0a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cc0a100] error while decoding MB 0 35, bytestream 2262 [h264 @ 0x55ff7cbaf640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cbaf640] error while decoding MB 0 25, bytestream 1604 [h264 @ 0x55ff7ccb8640] error while decoding MB 17 26, bytestr", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 491, "h264_decode_error_count": 1433, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:36:14+09:00", "last_error_summary": "[h264 @ 0x55ff7cbaf640] left block unavailable for requested intra mode [h264 @ 0x55ff7cbaf640] error while decoding MB 0 16, bytestream 3827 [h264 @ 0x55ff7cc0a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cc0a100] error while decoding MB 0 35, bytestream 2262 [h264 @ 0x55ff7cbaf640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cbaf640] error while decoding MB 0 25, bytestream 1604 [h264 @ 0x55ff7ccb8640] error while decoding MB 17 26, bytestr", "last_exit_at": "2026-07-01T10:36:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.601, "last_started_at": "2026-07-01T10:33:02+09:00", "pid": 97079, "process": {"exists": false, "pid": 97079}, "restart_count": 538, "restart_count_per_hour": 22, "rtsp_timeout_count": 408, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 539}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.667, "sample_count": 1721, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446204, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:36:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T10:36:14+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-01T10:36:14+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-01T10:36:14.066099+09:00", "mtime_epoch": 1782869774.0660987, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169725}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:36:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5055.492, "name": "celery", "pid": 267, "rss_mb": 5055.492, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 70, "pushed_frames": 70, "read_frames": 69, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.339, "pushed": 2.339, "read": 2.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.443, "count": 70, "max_ms": 38.638, "p95_ms": 35.477}, "gstreamer_push": {"avg_ms": 1.019, "count": 70, "max_ms": 1.149, "p95_ms": 1.121}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.077, "count": 70, "max_ms": 44.104, "p95_ms": 40.437}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.068, "count": 70, "max_ms": 94.748, "p95_ms": 84.003}, "yolo11_640_fire_11class": {"avg_ms": 49.296, "count": 70, "max_ms": 70.019, "p95_ms": 62.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.622, "count": 70, "max_ms": 83.544, "p95_ms": 78.464}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.347, "count": 70, "max_ms": 189.253, "p95_ms": 177.243}, "read_interval": {"avg_ms": 434.045, "count": 69, "max_ms": 18570.815, "p95_ms": 189.981}, "triton_wait": {"avg_ms": 63.426, "count": 70, "max_ms": 94.761, "p95_ms": 84.015}, "visualize": {"avg_ms": 5.11, "count": 70, "max_ms": 6.011, "p95_ms": 5.626}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12051}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10902.359, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 491, "h264_decode_error_count": 1433, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:36:14+09:00", "last_error_summary": "[h264 @ 0x55ff7cbaf640] left block unavailable for requested intra mode [h264 @ 0x55ff7cbaf640] error while decoding MB 0 16, bytestream 3827 [h264 @ 0x55ff7cc0a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cc0a100] error while decoding MB 0 35, bytestream 2262 [h264 @ 0x55ff7cbaf640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cbaf640] error while decoding MB 0 25, bytestream 1604 [h264 @ 0x55ff7ccb8640] error while decoding MB 17 26, bytestr", "last_exit_at": "2026-07-01T10:36:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.606, "last_started_at": "2026-07-01T10:33:02+09:00", "pid": 97079, "process": {"exists": false, "pid": 97079}, "restart_count": 538, "restart_count_per_hour": 22, "rtsp_timeout_count": 408, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 539}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.667, "sample_count": 1721, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446204, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:36:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T10:36:14+09:00", "last_frame_read_age_sec": 0.146, "last_frame_read_at": "2026-07-01T10:36:14+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-01T10:36:14.066099+09:00", "mtime_epoch": 1782869774.0660987, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169725}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:36:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5055.492, "name": "celery", "pid": 267, "rss_mb": 5055.492, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 70, "pushed_frames": 70, "read_frames": 69, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.338, "pushed": 2.338, "read": 2.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.443, "count": 70, "max_ms": 38.638, "p95_ms": 35.477}, "gstreamer_push": {"avg_ms": 1.019, "count": 70, "max_ms": 1.149, "p95_ms": 1.121}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.077, "count": 70, "max_ms": 44.104, "p95_ms": 40.437}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.068, "count": 70, "max_ms": 94.748, "p95_ms": 84.003}, "yolo11_640_fire_11class": {"avg_ms": 49.296, "count": 70, "max_ms": 70.019, "p95_ms": 62.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.622, "count": 70, "max_ms": 83.544, "p95_ms": 78.464}}, "truncated_count": 0}, "processing_total": {"avg_ms": 156.347, "count": 70, "max_ms": 189.253, "p95_ms": 177.243}, "read_interval": {"avg_ms": 434.045, "count": 69, "max_ms": 18570.815, "p95_ms": 189.981}, "triton_wait": {"avg_ms": 63.426, "count": 70, "max_ms": 94.761, "p95_ms": 84.015}, "visualize": {"avg_ms": 5.11, "count": 70, "max_ms": 6.011, "p95_ms": 5.626}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12051}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11005.094, "limit_mb": 8796093022207.996}, "detail": {"pid": 97186, "restart_count": 539, "start_count": 540}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 491, "h264_decode_error_count": 1433, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:36:14+09:00", "last_error_summary": "[h264 @ 0x55ff7cbaf640] left block unavailable for requested intra mode [h264 @ 0x55ff7cbaf640] error while decoding MB 0 16, bytestream 3827 [h264 @ 0x55ff7cc0a100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cc0a100] error while decoding MB 0 35, bytestream 2262 [h264 @ 0x55ff7cbaf640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ff7cbaf640] error while decoding MB 0 25, bytestream 1604 [h264 @ 0x55ff7ccb8640] error while decoding MB 17 26, bytestr", "last_exit_at": "2026-07-01T10:36:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:36:16+09:00", "pid": 97186, "process": {"exists": true, "fd_count": 5, "hwm_mb": 134.238, "name": "ffmpeg", "pid": 97186, "rss_mb": 134.238, "state": "S (sleeping)", "threads": 106}, "restart_count": 539, "restart_count_per_hour": 23, "rtsp_timeout_count": 408, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 540}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.721, "sample_count": 1692, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446204, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:36:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.291, "last_frame_pushed_at": "2026-07-01T10:36:14+09:00", "last_frame_read_age_sec": 4.428, "last_frame_read_at": "2026-07-01T10:36:14+09:00", "mjpeg": {"age_sec": 4.274, "exists": true, "mtime": "2026-07-01T10:36:14.214102+09:00", "mtime_epoch": 1782869774.2141016, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167635}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:36:18+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5055.492, "name": "celery", "pid": 267, "rss_mb": 5055.492, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 45, "pushed_frames": 45, "read_frames": 44, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.502, "pushed": 1.502, "read": 1.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.463, "count": 44, "max_ms": 38.638, "p95_ms": 35.8}, "gstreamer_push": {"avg_ms": 1.025, "count": 45, "max_ms": 1.149, "p95_ms": 1.132}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.967, "count": 44, "max_ms": 44.104, "p95_ms": 41.664}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.682, "count": 44, "max_ms": 84.003, "p95_ms": 80.703}, "yolo11_640_fire_11class": {"avg_ms": 49.123, "count": 44, "max_ms": 62.693, "p95_ms": 59.728}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.65, "count": 44, "max_ms": 79.044, "p95_ms": 77.865}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.401, "count": 45, "max_ms": 180.862, "p95_ms": 173.69}, "read_interval": {"avg_ms": 583.867, "count": 44, "max_ms": 18570.815, "p95_ms": 188.892}, "triton_wait": {"avg_ms": 63.243, "count": 44, "max_ms": 84.015, "p95_ms": 80.715}, "visualize": {"avg_ms": 5.012, "count": 44, "max_ms": 6.011, "p95_ms": 5.607}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10903.91, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5574bfd0e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfd0e880] error while decoding MB 0 29, bytestream 740 [h264 @ 0x5574bfc91f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfc91f80] error while decoding MB 0 2, bytestream 44011 [h264 @ 0x5574bfbbbb40] cabac decode of qscale diff failed at 45 18 [h264 @ 0x5574bfbbbb40] error while decoding MB 45 18, bytestream 53857 [h264 @ 0x5574c0611400] error while decoding MB 28 31, bytestream", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 492, "h264_decode_error_count": 1438, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:38:37+09:00", "last_error_summary": "[h264 @ 0x5574bfd0e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfd0e880] error while decoding MB 0 29, bytestream 740 [h264 @ 0x5574bfc91f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfc91f80] error while decoding MB 0 2, bytestream 44011 [h264 @ 0x5574bfbbbb40] cabac decode of qscale diff failed at 45 18 [h264 @ 0x5574bfbbbb40] error while decoding MB 45 18, bytestream 53857 [h264 @ 0x5574c0611400] error while decoding MB 28 31, bytestream", "last_exit_at": "2026-07-01T10:36:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 141.069, "last_started_at": "2026-07-01T10:36:16+09:00", "pid": 97186, "process": {"exists": false, "pid": 97186}, "restart_count": 539, "restart_count_per_hour": 22, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 540}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.086, "sample_count": 1498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446889, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:38:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T10:38:37+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T10:38:37+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T10:38:37.512874+09:00", "mtime_epoch": 1782869917.512874, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183564}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:38:37+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5055.75, "name": "celery", "pid": 267, "rss_mb": 5055.75, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 64, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.257, "pushed": 2.257, "read": 2.293}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.845, "count": 63, "max_ms": 35.549, "p95_ms": 34.488}, "gstreamer_push": {"avg_ms": 1.102, "count": 63, "max_ms": 1.186, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.03, "count": 63, "max_ms": 39.552, "p95_ms": 36.574}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.044, "count": 63, "max_ms": 90.007, "p95_ms": 81.296}, "yolo11_640_fire_11class": {"avg_ms": 48.169, "count": 63, "max_ms": 68.316, "p95_ms": 61.319}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.075, "count": 63, "max_ms": 86.415, "p95_ms": 80.758}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.992, "count": 63, "max_ms": 159.909, "p95_ms": 153.726}, "read_interval": {"avg_ms": 484.377, "count": 64, "max_ms": 18866.269, "p95_ms": 171.081}, "triton_wait": {"avg_ms": 61.078, "count": 63, "max_ms": 90.02, "p95_ms": 81.308}, "visualize": {"avg_ms": 4.276, "count": 63, "max_ms": 5.328, "p95_ms": 4.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.913, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10903.352, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:38:37+09:00", "last_error_summary": "[h264 @ 0x5574bfd0e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfd0e880] error while decoding MB 0 29, bytestream 740 [h264 @ 0x5574bfc91f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfc91f80] error while decoding MB 0 2, bytestream 44011 [h264 @ 0x5574bfbbbb40] cabac decode of qscale diff failed at 45 18 [h264 @ 0x5574bfbbbb40] error while decoding MB 45 18, bytestream 53857 [h264 @ 0x5574c0611400] error while decoding MB 28 31, bytestream", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 492, "h264_decode_error_count": 1438, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:38:37+09:00", "last_error_summary": "[h264 @ 0x5574bfd0e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfd0e880] error while decoding MB 0 29, bytestream 740 [h264 @ 0x5574bfc91f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfc91f80] error while decoding MB 0 2, bytestream 44011 [h264 @ 0x5574bfbbbb40] cabac decode of qscale diff failed at 45 18 [h264 @ 0x5574bfbbbb40] error while decoding MB 45 18, bytestream 53857 [h264 @ 0x5574c0611400] error while decoding MB 28 31, bytestream", "last_exit_at": "2026-07-01T10:38:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 141.181, "last_started_at": "2026-07-01T10:36:16+09:00", "pid": 97186, "process": {"exists": false, "pid": 97186}, "restart_count": 539, "restart_count_per_hour": 22, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 540}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.086, "sample_count": 1498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446890, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:38:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T10:38:37+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-01T10:38:37+09:00", "mjpeg": {"age_sec": 0.155, "exists": true, "mtime": "2026-07-01T10:38:37.512874+09:00", "mtime_epoch": 1782869917.512874, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183564}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:38:37+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5055.75, "name": "celery", "pid": 267, "rss_mb": 5055.75, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.284, "pushed": 2.284, "read": 2.284}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.814, "count": 64, "max_ms": 35.549, "p95_ms": 34.488}, "gstreamer_push": {"avg_ms": 1.1, "count": 64, "max_ms": 1.186, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.026, "count": 64, "max_ms": 39.552, "p95_ms": 36.574}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.102, "count": 64, "max_ms": 90.007, "p95_ms": 81.296}, "yolo11_640_fire_11class": {"avg_ms": 48.097, "count": 64, "max_ms": 68.316, "p95_ms": 61.319}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.039, "count": 64, "max_ms": 86.415, "p95_ms": 80.758}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.404, "count": 64, "max_ms": 159.909, "p95_ms": 153.809}, "read_interval": {"avg_ms": 484.377, "count": 64, "max_ms": 18866.269, "p95_ms": 171.081}, "triton_wait": {"avg_ms": 61.135, "count": 64, "max_ms": 90.02, "p95_ms": 81.308}, "visualize": {"avg_ms": 4.291, "count": 64, "max_ms": 5.328, "p95_ms": 4.533}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.025, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10903.344, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 492, "h264_decode_error_count": 1438, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:38:37+09:00", "last_error_summary": "[h264 @ 0x5574bfd0e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfd0e880] error while decoding MB 0 29, bytestream 740 [h264 @ 0x5574bfc91f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfc91f80] error while decoding MB 0 2, bytestream 44011 [h264 @ 0x5574bfbbbb40] cabac decode of qscale diff failed at 45 18 [h264 @ 0x5574bfbbbb40] error while decoding MB 45 18, bytestream 53857 [h264 @ 0x5574c0611400] error while decoding MB 28 31, bytestream", "last_exit_at": "2026-07-01T10:38:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 141.185, "last_started_at": "2026-07-01T10:36:16+09:00", "pid": 97186, "process": {"exists": false, "pid": 97186}, "restart_count": 539, "restart_count_per_hour": 22, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 540}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.086, "sample_count": 1498, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446890, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:38:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T10:38:37+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T10:38:37+09:00", "mjpeg": {"age_sec": 0.159, "exists": true, "mtime": "2026-07-01T10:38:37.512874+09:00", "mtime_epoch": 1782869917.512874, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183564}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:38:37+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5055.75, "name": "celery", "pid": 267, "rss_mb": 5055.75, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.283, "pushed": 2.283, "read": 2.283}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.814, "count": 64, "max_ms": 35.549, "p95_ms": 34.488}, "gstreamer_push": {"avg_ms": 1.1, "count": 64, "max_ms": 1.186, "p95_ms": 1.15}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.026, "count": 64, "max_ms": 39.552, "p95_ms": 36.574}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.102, "count": 64, "max_ms": 90.007, "p95_ms": 81.296}, "yolo11_640_fire_11class": {"avg_ms": 48.097, "count": 64, "max_ms": 68.316, "p95_ms": 61.319}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.039, "count": 64, "max_ms": 86.415, "p95_ms": 80.758}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.404, "count": 64, "max_ms": 159.909, "p95_ms": 153.809}, "read_interval": {"avg_ms": 484.377, "count": 64, "max_ms": 18866.269, "p95_ms": 171.081}, "triton_wait": {"avg_ms": 61.135, "count": 64, "max_ms": 90.02, "p95_ms": 81.308}, "visualize": {"avg_ms": 4.291, "count": 64, "max_ms": 5.328, "p95_ms": 4.533}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.028, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11008.348, "limit_mb": 8796093022207.996}, "detail": {"pid": 97293, "restart_count": 540, "start_count": 541}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 492, "h264_decode_error_count": 1438, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:38:37+09:00", "last_error_summary": "[h264 @ 0x5574bfd0e880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfd0e880] error while decoding MB 0 29, bytestream 740 [h264 @ 0x5574bfc91f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5574bfc91f80] error while decoding MB 0 2, bytestream 44011 [h264 @ 0x5574bfbbbb40] cabac decode of qscale diff failed at 45 18 [h264 @ 0x5574bfbbbb40] error while decoding MB 45 18, bytestream 53857 [h264 @ 0x5574c0611400] error while decoding MB 28 31, bytestream", "last_exit_at": "2026-07-01T10:38:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:38:39+09:00", "pid": 97293, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.301, "name": "ffmpeg", "pid": 97293, "rss_mb": 136.301, "state": "S (sleeping)", "threads": 106}, "restart_count": 540, "restart_count_per_hour": 23, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 541}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.358, "sample_count": 1469, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1446890, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:38:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.293, "last_frame_pushed_at": "2026-07-01T10:38:37+09:00", "last_frame_read_age_sec": 4.453, "last_frame_read_at": "2026-07-01T10:38:37+09:00", "mjpeg": {"age_sec": 4.277, "exists": true, "mtime": "2026-07-01T10:38:37.680877+09:00", "mtime_epoch": 1782869917.680877, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:38:41+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5055.75, "name": "celery", "pid": 267, "rss_mb": 5055.75, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.638, "pushed": 1.638, "read": 1.605}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.705, "count": 49, "max_ms": 35.549, "p95_ms": 34.349}, "gstreamer_push": {"avg_ms": 1.096, "count": 49, "max_ms": 1.159, "p95_ms": 1.146}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.102, "count": 49, "max_ms": 39.552, "p95_ms": 36.961}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.946, "count": 49, "max_ms": 84.238, "p95_ms": 81.265}, "yolo11_640_fire_11class": {"avg_ms": 47.761, "count": 49, "max_ms": 65.47, "p95_ms": 60.635}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.217, "count": 49, "max_ms": 81.724, "p95_ms": 77.43}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.398, "count": 49, "max_ms": 158.349, "p95_ms": 153.726}, "read_interval": {"avg_ms": 533.153, "count": 48, "max_ms": 18866.269, "p95_ms": 164.536}, "triton_wait": {"avg_ms": 60.986, "count": 49, "max_ms": 84.25, "p95_ms": 81.278}, "visualize": {"avg_ms": 4.302, "count": 49, "max_ms": 5.328, "p95_ms": 4.558}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13727}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10841.844, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c0a8707c80] error while decoding MB 98 33, bytestream -13 [in#0/rtsp @ 0x55c0a8571e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55c0a8571e00] Error retrieving a packet from demuxer: Input/output error", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 213, "h264_decode_error_count": 219, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:39:35+09:00", "last_error_summary": "[h264 @ 0x55c0a8707c80] error while decoding MB 98 33, bytestream -13 [in#0/rtsp @ 0x55c0a8571e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55c0a8571e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:32:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 448.054, "last_started_at": "2026-07-01T10:32:07+09:00", "pid": 96927, "process": {"exists": false, "pid": 96927}, "restart_count": 212, "restart_count_per_hour": 6, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 213}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.745, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1647299, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:39:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.085, "last_frame_pushed_at": "2026-07-01T10:39:35+09:00", "last_frame_read_age_sec": 0.074, "last_frame_read_at": "2026-07-01T10:39:35+09:00", "mjpeg": {"age_sec": 0.068, "exists": true, "mtime": "2026-07-01T10:39:35.461977+09:00", "mtime_epoch": 1782869975.4619768, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 213825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:39:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1841.477, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.745, "pushed": 0.745, "read": 0.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 22.906, "count": 13, "max_ms": 28.605, "p95_ms": 28.605}, "gstreamer_push": {"avg_ms": 1.198, "count": 13, "max_ms": 1.279, "p95_ms": 1.279}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.758, "count": 13, "max_ms": 50.967, "p95_ms": 50.967}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.158, "count": 13, "max_ms": 80.447, "p95_ms": 80.447}, "yolo11_640_fire_11class": {"avg_ms": 47.986, "count": 13, "max_ms": 51.874, "p95_ms": 51.874}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.571, "count": 13, "max_ms": 76.203, "p95_ms": 76.203}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.135, "count": 13, "max_ms": 108.408, "p95_ms": 108.408}, "read_interval": {"avg_ms": 2427.229, "count": 14, "max_ms": 16603.807, "p95_ms": 16603.807}, "triton_wait": {"avg_ms": 58.171, "count": 13, "max_ms": 80.46, "p95_ms": 80.46}, "visualize": {"avg_ms": 3.387, "count": 13, "max_ms": 3.871, "p95_ms": 3.871}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.452, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13727}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10841.707, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:39:35+09:00", "last_error_summary": "[h264 @ 0x55c0a8707c80] error while decoding MB 98 33, bytestream -13 [in#0/rtsp @ 0x55c0a8571e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55c0a8571e00] Error retrieving a packet from demuxer: Input/output error", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 213, "h264_decode_error_count": 219, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:39:35+09:00", "last_error_summary": "[h264 @ 0x55c0a8707c80] error while decoding MB 98 33, bytestream -13 [in#0/rtsp @ 0x55c0a8571e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55c0a8571e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:39:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 448.077, "last_started_at": "2026-07-01T10:32:07+09:00", "pid": 96927, "process": {"exists": false, "pid": 96927}, "restart_count": 212, "restart_count_per_hour": 6, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 213}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.745, "sample_count": 2561, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1647300, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:39:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T10:39:35+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-01T10:39:35+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T10:39:35.461977+09:00", "mtime_epoch": 1782869975.4619768, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 213825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:39:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1841.477, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.801, "pushed": 0.801, "read": 0.801}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.06, "count": 14, "max_ms": 28.605, "p95_ms": 28.605}, "gstreamer_push": {"avg_ms": 1.207, "count": 14, "max_ms": 1.323, "p95_ms": 1.323}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.724, "count": 14, "max_ms": 50.967, "p95_ms": 50.967}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.879, "count": 14, "max_ms": 80.447, "p95_ms": 80.447}, "yolo11_640_fire_11class": {"avg_ms": 47.957, "count": 14, "max_ms": 51.874, "p95_ms": 51.874}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.407, "count": 14, "max_ms": 76.203, "p95_ms": 76.203}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.166, "count": 14, "max_ms": 108.408, "p95_ms": 108.408}, "read_interval": {"avg_ms": 2427.229, "count": 14, "max_ms": 16603.807, "p95_ms": 16603.807}, "triton_wait": {"avg_ms": 57.892, "count": 14, "max_ms": 80.46, "p95_ms": 80.46}, "visualize": {"avg_ms": 3.513, "count": 14, "max_ms": 5.156, "p95_ms": 5.156}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.475, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13727}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10841.629, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 213, "h264_decode_error_count": 219, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:39:35+09:00", "last_error_summary": "[h264 @ 0x55c0a8707c80] error while decoding MB 98 33, bytestream -13 [in#0/rtsp @ 0x55c0a8571e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55c0a8571e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:39:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 448.084, "last_started_at": "2026-07-01T10:32:07+09:00", "pid": 96927, "process": {"exists": false, "pid": 96927}, "restart_count": 212, "restart_count_per_hour": 6, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 213}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.745, "sample_count": 2561, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1647300, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:39:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T10:39:35+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T10:39:35+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-01T10:39:35.461977+09:00", "mtime_epoch": 1782869975.4619768, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 213825}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:39:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1841.477, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.801, "pushed": 0.801, "read": 0.801}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.06, "count": 14, "max_ms": 28.605, "p95_ms": 28.605}, "gstreamer_push": {"avg_ms": 1.207, "count": 14, "max_ms": 1.323, "p95_ms": 1.323}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.724, "count": 14, "max_ms": 50.967, "p95_ms": 50.967}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.879, "count": 14, "max_ms": 80.447, "p95_ms": 80.447}, "yolo11_640_fire_11class": {"avg_ms": 47.957, "count": 14, "max_ms": 51.874, "p95_ms": 51.874}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.407, "count": 14, "max_ms": 76.203, "p95_ms": 76.203}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.166, "count": 14, "max_ms": 108.408, "p95_ms": 108.408}, "read_interval": {"avg_ms": 2427.229, "count": 14, "max_ms": 16603.807, "p95_ms": 16603.807}, "triton_wait": {"avg_ms": 57.892, "count": 14, "max_ms": 80.46, "p95_ms": 80.46}, "visualize": {"avg_ms": 3.513, "count": 14, "max_ms": 5.156, "p95_ms": 5.156}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.482, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 73, "tracking_summary": 13727}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 11021.242, "limit_mb": 8796093022207.996}, "detail": {"pid": 97400, "restart_count": 213, "start_count": 214}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 213, "h264_decode_error_count": 219, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:39:35+09:00", "last_error_summary": "[h264 @ 0x55c0a8707c80] error while decoding MB 98 33, bytestream -13 [in#0/rtsp @ 0x55c0a8571e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55c0a8571e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:39:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:39:37+09:00", "pid": 97400, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.566, "name": "ffmpeg", "pid": 97400, "rss_mb": 209.566, "state": "S (sleeping)", "threads": 106}, "restart_count": 213, "restart_count_per_hour": 7, "rtsp_timeout_count": 350, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 214}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.818, "sample_count": 2520, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1647300, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:39:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.198, "last_frame_pushed_at": "2026-07-01T10:39:35+09:00", "last_frame_read_age_sec": 4.289, "last_frame_read_at": "2026-07-01T10:39:35+09:00", "mjpeg": {"age_sec": 4.179, "exists": true, "mtime": "2026-07-01T10:39:35.565979+09:00", "mtime_epoch": 1782869975.5659788, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211816}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:39:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2072.379, "name": "celery", "pid": 237, "rss_mb": 1841.477, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.646, "pushed": 0.646, "read": 0.646}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.06, "count": 14, "max_ms": 28.605, "p95_ms": 28.605}, "gstreamer_push": {"avg_ms": 1.207, "count": 14, "max_ms": 1.323, "p95_ms": 1.323}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.724, "count": 14, "max_ms": 50.967, "p95_ms": 50.967}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.879, "count": 14, "max_ms": 80.447, "p95_ms": 80.447}, "yolo11_640_fire_11class": {"avg_ms": 47.957, "count": 14, "max_ms": 51.874, "p95_ms": 51.874}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.407, "count": 14, "max_ms": 76.203, "p95_ms": 76.203}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.166, "count": 14, "max_ms": 108.408, "p95_ms": 108.408}, "read_interval": {"avg_ms": 2427.229, "count": 14, "max_ms": 16603.807, "p95_ms": 16603.807}, "triton_wait": {"avg_ms": 57.892, "count": 14, "max_ms": 80.46, "p95_ms": 80.46}, "visualize": {"avg_ms": 3.513, "count": 14, "max_ms": 5.156, "p95_ms": 5.156}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.667, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12062}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11186.707, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557be3810100] error while decoding MB 50 33, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 493, "h264_decode_error_count": 1439, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:40:51+09:00", "last_error_summary": "[h264 @ 0x557be3810100] error while decoding MB 50 33, bytestream -10", "last_exit_at": "2026-07-01T10:38:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.36, "last_started_at": "2026-07-01T10:38:39+09:00", "pid": 97293, "process": {"exists": false, "pid": 97293}, "restart_count": 540, "restart_count_per_hour": 22, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 541}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.945, "sample_count": 1375, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1447551, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:40:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T10:40:51+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T10:40:51+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T10:40:51.271406+09:00", "mtime_epoch": 1782870051.2714062, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172318}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:40:51+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5056.008, "name": "celery", "pid": 267, "rss_mb": 5056.008, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 97, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.239, "pushed": 3.239, "read": 3.239}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.25, "count": 97, "max_ms": 36.6, "p95_ms": 33.65}, "gstreamer_push": {"avg_ms": 1.084, "count": 97, "max_ms": 1.176, "p95_ms": 1.154}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.151, "count": 97, "max_ms": 36.249, "p95_ms": 35.701}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.843, "count": 97, "max_ms": 87.096, "p95_ms": 78.585}, "yolo11_640_fire_11class": {"avg_ms": 45.684, "count": 97, "max_ms": 66.311, "p95_ms": 60.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.389, "count": 97, "max_ms": 82.972, "p95_ms": 78.225}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.915, "count": 97, "max_ms": 180.801, "p95_ms": 157.851}, "read_interval": {"avg_ms": 309.725, "count": 97, "max_ms": 10009.982, "p95_ms": 192.292}, "triton_wait": {"avg_ms": 59.939, "count": 97, "max_ms": 87.108, "p95_ms": 80.361}, "visualize": {"avg_ms": 4.472, "count": 97, "max_ms": 5.557, "p95_ms": 5.429}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12062}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11186.469, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:40:51+09:00", "last_error_summary": "[h264 @ 0x557be3810100] error while decoding MB 50 33, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 493, "h264_decode_error_count": 1439, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:40:51+09:00", "last_error_summary": "[h264 @ 0x557be3810100] error while decoding MB 50 33, bytestream -10", "last_exit_at": "2026-07-01T10:40:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.463, "last_started_at": "2026-07-01T10:38:39+09:00", "pid": 97293, "process": {"exists": false, "pid": 97293}, "restart_count": 540, "restart_count_per_hour": 22, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 541}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.945, "sample_count": 1375, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1447552, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:40:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T10:40:51+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-01T10:40:51+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-01T10:40:51.271406+09:00", "mtime_epoch": 1782870051.2714062, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172318}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:40:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5056.008, "name": "celery", "pid": 267, "rss_mb": 5056.008, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.243, "pushed": 3.243, "read": 3.21}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.249, "count": 97, "max_ms": 36.6, "p95_ms": 33.65}, "gstreamer_push": {"avg_ms": 1.083, "count": 97, "max_ms": 1.176, "p95_ms": 1.154}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.104, "count": 97, "max_ms": 36.249, "p95_ms": 34.48}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.617, "count": 97, "max_ms": 87.096, "p95_ms": 76.102}, "yolo11_640_fire_11class": {"avg_ms": 45.544, "count": 97, "max_ms": 66.311, "p95_ms": 60.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.156, "count": 97, "max_ms": 82.972, "p95_ms": 78.225}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.964, "count": 97, "max_ms": 180.801, "p95_ms": 157.851}, "read_interval": {"avg_ms": 311.304, "count": 96, "max_ms": 10009.982, "p95_ms": 192.292}, "triton_wait": {"avg_ms": 59.712, "count": 97, "max_ms": 87.108, "p95_ms": 80.361}, "visualize": {"avg_ms": 4.487, "count": 97, "max_ms": 5.637, "p95_ms": 5.456}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12062}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11186.367, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 493, "h264_decode_error_count": 1439, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:40:51+09:00", "last_error_summary": "[h264 @ 0x557be3810100] error while decoding MB 50 33, bytestream -10", "last_exit_at": "2026-07-01T10:40:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.467, "last_started_at": "2026-07-01T10:38:39+09:00", "pid": 97293, "process": {"exists": false, "pid": 97293}, "restart_count": 540, "restart_count_per_hour": 22, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 541}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.945, "sample_count": 1375, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1447552, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:40:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T10:40:51+09:00", "last_frame_read_age_sec": 0.16, "last_frame_read_at": "2026-07-01T10:40:51+09:00", "mjpeg": {"age_sec": 0.152, "exists": true, "mtime": "2026-07-01T10:40:51.271406+09:00", "mtime_epoch": 1782870051.2714062, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172318}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:40:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5056.008, "name": "celery", "pid": 267, "rss_mb": 5056.008, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.243, "pushed": 3.243, "read": 3.209}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.249, "count": 97, "max_ms": 36.6, "p95_ms": 33.65}, "gstreamer_push": {"avg_ms": 1.083, "count": 97, "max_ms": 1.176, "p95_ms": 1.154}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.104, "count": 97, "max_ms": 36.249, "p95_ms": 34.48}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.617, "count": 97, "max_ms": 87.096, "p95_ms": 76.102}, "yolo11_640_fire_11class": {"avg_ms": 45.544, "count": 97, "max_ms": 66.311, "p95_ms": 60.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.156, "count": 97, "max_ms": 82.972, "p95_ms": 78.225}}, "truncated_count": 0}, "processing_total": {"avg_ms": 132.964, "count": 97, "max_ms": 180.801, "p95_ms": 157.851}, "read_interval": {"avg_ms": 311.304, "count": 96, "max_ms": 10009.982, "p95_ms": 192.292}, "triton_wait": {"avg_ms": 59.712, "count": 97, "max_ms": 87.108, "p95_ms": 80.361}, "visualize": {"avg_ms": 4.487, "count": 97, "max_ms": 5.637, "p95_ms": 5.456}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12062}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11291.656, "limit_mb": 8796093022207.996}, "detail": {"pid": 97672, "restart_count": 541, "start_count": 542}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 493, "h264_decode_error_count": 1439, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:40:51+09:00", "last_error_summary": "[h264 @ 0x557be3810100] error while decoding MB 50 33, bytestream -10", "last_exit_at": "2026-07-01T10:40:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:40:53+09:00", "pid": 97672, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.672, "name": "ffmpeg", "pid": 97672, "rss_mb": 136.672, "state": "S (sleeping)", "threads": 106}, "restart_count": 541, "restart_count_per_hour": 23, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 542}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.945, "sample_count": 1359, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1447552, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:40:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.375, "last_frame_pushed_at": "2026-07-01T10:40:51+09:00", "last_frame_read_age_sec": 4.528, "last_frame_read_at": "2026-07-01T10:40:51+09:00", "mjpeg": {"age_sec": 4.357, "exists": true, "mtime": "2026-07-01T10:40:51.435409+09:00", "mtime_epoch": 1782870051.4354093, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166474}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:40:55+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5056.008, "name": "celery", "pid": 267, "rss_mb": 5056.008, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 69, "pushed_frames": 69, "read_frames": 68, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.304, "pushed": 2.304, "read": 2.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 29.69, "count": 69, "max_ms": 36.6, "p95_ms": 33.615}, "gstreamer_push": {"avg_ms": 1.09, "count": 69, "max_ms": 1.176, "p95_ms": 1.154}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.09, "count": 69, "max_ms": 36.249, "p95_ms": 34.48}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.775, "count": 69, "max_ms": 87.09, "p95_ms": 69.56}, "yolo11_640_fire_11class": {"avg_ms": 44.225, "count": 69, "max_ms": 66.311, "p95_ms": 50.721}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.431, "count": 69, "max_ms": 79.08, "p95_ms": 58.738}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.0, "count": 69, "max_ms": 158.3, "p95_ms": 151.341}, "read_interval": {"avg_ms": 376.163, "count": 68, "max_ms": 10009.982, "p95_ms": 280.847}, "triton_wait": {"avg_ms": 57.788, "count": 69, "max_ms": 87.103, "p95_ms": 69.571}, "visualize": {"avg_ms": 4.446, "count": 69, "max_ms": 5.637, "p95_ms": 5.429}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 77, "tracking_summary": 13730}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 11110.547, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x564cdd9bc540] error while decoding MB 47 40, bytestream -23 [in#0/rtsp @ 0x564cdd0afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564cdd0afe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 214, "h264_decode_error_count": 220, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:41:04+09:00", "last_error_summary": "[h264 @ 0x564cdd9bc540] error while decoding MB 47 40, bytestream -23 [in#0/rtsp @ 0x564cdd0afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564cdd0afe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:39:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 86.815, "last_started_at": "2026-07-01T10:39:37+09:00", "pid": 97400, "process": {"exists": false, "pid": 97400}, "restart_count": 213, "restart_count_per_hour": 7, "rtsp_timeout_count": 352, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 214}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.117, "sample_count": 2182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1647689, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:41:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-01T10:41:04+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-01T10:41:04+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T10:41:04.511654+09:00", "mtime_epoch": 1782870064.5116544, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:41:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 2096.277, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.72, "pushed": 0.72, "read": 0.775}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.008, "count": 13, "max_ms": 28.851, "p95_ms": 28.851}, "gstreamer_push": {"avg_ms": 1.077, "count": 13, "max_ms": 1.216, "p95_ms": 1.216}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.967, "count": 13, "max_ms": 57.503, "p95_ms": 57.503}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.955, "count": 13, "max_ms": 82.545, "p95_ms": 82.545}, "yolo11_640_fire_11class": {"avg_ms": 48.195, "count": 13, "max_ms": 59.633, "p95_ms": 59.633}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.657, "count": 13, "max_ms": 79.19, "p95_ms": 79.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.838, "count": 13, "max_ms": 146.455, "p95_ms": 146.455}, "read_interval": {"avg_ms": 2547.591, "count": 14, "max_ms": 17670.544, "p95_ms": 17670.544}, "triton_wait": {"avg_ms": 56.968, "count": 13, "max_ms": 82.555, "p95_ms": 82.555}, "visualize": {"avg_ms": 4.013, "count": 13, "max_ms": 5.643, "p95_ms": 5.643}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.055, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 77, "tracking_summary": 13730}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 11109.609, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:41:04+09:00", "last_error_summary": "[h264 @ 0x564cdd9bc540] error while decoding MB 47 40, bytestream -23 [in#0/rtsp @ 0x564cdd0afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564cdd0afe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 214, "h264_decode_error_count": 220, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:41:04+09:00", "last_error_summary": "[h264 @ 0x564cdd9bc540] error while decoding MB 47 40, bytestream -23 [in#0/rtsp @ 0x564cdd0afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564cdd0afe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:41:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 86.926, "last_started_at": "2026-07-01T10:39:37+09:00", "pid": 97400, "process": {"exists": false, "pid": 97400}, "restart_count": 213, "restart_count_per_hour": 7, "rtsp_timeout_count": 352, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 214}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.135, "sample_count": 2182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1647690, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:41:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T10:41:04+09:00", "last_frame_read_age_sec": 0.169, "last_frame_read_at": "2026-07-01T10:41:04+09:00", "mjpeg": {"age_sec": 0.159, "exists": true, "mtime": "2026-07-01T10:41:04.511654+09:00", "mtime_epoch": 1782870064.5116544, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:41:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 2096.277, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.771, "pushed": 0.771, "read": 0.771}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.099, "count": 14, "max_ms": 28.851, "p95_ms": 28.851}, "gstreamer_push": {"avg_ms": 1.068, "count": 14, "max_ms": 1.216, "p95_ms": 1.216}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.932, "count": 14, "max_ms": 57.503, "p95_ms": 57.503}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.143, "count": 14, "max_ms": 82.545, "p95_ms": 82.545}, "yolo11_640_fire_11class": {"avg_ms": 48.388, "count": 14, "max_ms": 59.633, "p95_ms": 59.633}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.965, "count": 14, "max_ms": 79.19, "p95_ms": 79.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.392, "count": 14, "max_ms": 164.596, "p95_ms": 164.596}, "read_interval": {"avg_ms": 2547.591, "count": 14, "max_ms": 17670.544, "p95_ms": 17670.544}, "triton_wait": {"avg_ms": 58.156, "count": 14, "max_ms": 82.555, "p95_ms": 82.555}, "visualize": {"avg_ms": 4.071, "count": 14, "max_ms": 5.643, "p95_ms": 5.643}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.165, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 77, "tracking_summary": 13730}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 11109.602, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 214, "h264_decode_error_count": 220, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:41:04+09:00", "last_error_summary": "[h264 @ 0x564cdd9bc540] error while decoding MB 47 40, bytestream -23 [in#0/rtsp @ 0x564cdd0afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564cdd0afe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:41:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 86.929, "last_started_at": "2026-07-01T10:39:37+09:00", "pid": 97400, "process": {"exists": false, "pid": 97400}, "restart_count": 213, "restart_count_per_hour": 7, "rtsp_timeout_count": 352, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 214}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.135, "sample_count": 2182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1647690, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:41:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T10:41:04+09:00", "last_frame_read_age_sec": 0.173, "last_frame_read_at": "2026-07-01T10:41:04+09:00", "mjpeg": {"age_sec": 0.162, "exists": true, "mtime": "2026-07-01T10:41:04.511654+09:00", "mtime_epoch": 1782870064.5116544, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 211295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:41:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 2096.277, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.771, "pushed": 0.771, "read": 0.771}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.099, "count": 14, "max_ms": 28.851, "p95_ms": 28.851}, "gstreamer_push": {"avg_ms": 1.068, "count": 14, "max_ms": 1.216, "p95_ms": 1.216}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.932, "count": 14, "max_ms": 57.503, "p95_ms": 57.503}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.143, "count": 14, "max_ms": 82.545, "p95_ms": 82.545}, "yolo11_640_fire_11class": {"avg_ms": 48.388, "count": 14, "max_ms": 59.633, "p95_ms": 59.633}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.965, "count": 14, "max_ms": 79.19, "p95_ms": 79.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.392, "count": 14, "max_ms": 164.596, "p95_ms": 164.596}, "read_interval": {"avg_ms": 2547.591, "count": 14, "max_ms": 17670.544, "p95_ms": 17670.544}, "triton_wait": {"avg_ms": 58.156, "count": 14, "max_ms": 82.555, "p95_ms": 82.555}, "visualize": {"avg_ms": 4.071, "count": 14, "max_ms": 5.643, "p95_ms": 5.643}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.168, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 77, "tracking_summary": 13730}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 11225.801, "limit_mb": 8796093022207.996}, "detail": {"pid": 97779, "restart_count": 214, "start_count": 215}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 214, "h264_decode_error_count": 220, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:41:04+09:00", "last_error_summary": "[h264 @ 0x564cdd9bc540] error while decoding MB 47 40, bytestream -23 [in#0/rtsp @ 0x564cdd0afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564cdd0afe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:41:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:41:07+09:00", "pid": 97779, "process": {"exists": true, "fd_count": 5, "hwm_mb": 146.766, "name": "ffmpeg", "pid": 97779, "rss_mb": 146.766, "state": "R (running)", "threads": 49}, "restart_count": 214, "restart_count_per_hour": 8, "rtsp_timeout_count": 352, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 215}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.284, "sample_count": 2133, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1647690, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:41:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.613, "last_frame_pushed_at": "2026-07-01T10:41:04+09:00", "last_frame_read_age_sec": 4.779, "last_frame_read_at": "2026-07-01T10:41:04+09:00", "mjpeg": {"age_sec": 4.596, "exists": true, "mtime": "2026-07-01T10:41:04.683658+09:00", "mtime_epoch": 1782870064.6836576, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 214954}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T10:41:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 2096.277, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.615, "pushed": 0.615, "read": 0.615}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 23.099, "count": 14, "max_ms": 28.851, "p95_ms": 28.851}, "gstreamer_push": {"avg_ms": 1.068, "count": 14, "max_ms": 1.216, "p95_ms": 1.216}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.932, "count": 14, "max_ms": 57.503, "p95_ms": 57.503}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.143, "count": 14, "max_ms": 82.545, "p95_ms": 82.545}, "yolo11_640_fire_11class": {"avg_ms": 48.388, "count": 14, "max_ms": 59.633, "p95_ms": 59.633}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.965, "count": 14, "max_ms": 79.19, "p95_ms": 79.19}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.392, "count": 14, "max_ms": 164.596, "p95_ms": 164.596}, "read_interval": {"avg_ms": 2547.591, "count": 14, "max_ms": 17670.544, "p95_ms": 17670.544}, "triton_wait": {"avg_ms": 58.156, "count": 14, "max_ms": 82.555, "p95_ms": 82.555}, "visualize": {"avg_ms": 4.071, "count": 14, "max_ms": 5.643, "p95_ms": 5.643}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.775, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12066}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10988.031, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559c493537c0] error while decoding MB 46 26, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 494, "h264_decode_error_count": 1440, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:42:32+09:00", "last_error_summary": "[h264 @ 0x559c493537c0] error while decoding MB 46 26, bytestream -5", "last_exit_at": "2026-07-01T10:40:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 98.41, "last_started_at": "2026-07-01T10:40:53+09:00", "pid": 97672, "process": {"exists": false, "pid": 97672}, "restart_count": 541, "restart_count_per_hour": 22, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 542}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.905, "sample_count": 1369, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448002, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:42:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T10:42:32+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T10:42:32+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T10:42:32.157289+09:00", "mtime_epoch": 1782870152.1572886, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:42:32+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5056.266, "name": "celery", "pid": 267, "rss_mb": 5056.266, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 47, "pushed_frames": 47, "read_frames": 47, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.572, "pushed": 1.572, "read": 1.572}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.114, "count": 47, "max_ms": 53.003, "p95_ms": 35.999}, "gstreamer_push": {"avg_ms": 1.03, "count": 47, "max_ms": 1.212, "p95_ms": 1.136}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.449, "count": 47, "max_ms": 41.904, "p95_ms": 39.468}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.601, "count": 47, "max_ms": 87.996, "p95_ms": 81.075}, "yolo11_640_fire_11class": {"avg_ms": 47.675, "count": 47, "max_ms": 66.044, "p95_ms": 62.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.552, "count": 47, "max_ms": 83.044, "p95_ms": 77.579}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.642, "count": 47, "max_ms": 185.81, "p95_ms": 174.905}, "read_interval": {"avg_ms": 638.376, "count": 47, "max_ms": 14775.155, "p95_ms": 527.694}, "triton_wait": {"avg_ms": 62.795, "count": 47, "max_ms": 88.008, "p95_ms": 82.099}, "visualize": {"avg_ms": 5.036, "count": 47, "max_ms": 6.423, "p95_ms": 5.537}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12066}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10986.715, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:42:32+09:00", "last_error_summary": "[h264 @ 0x559c493537c0] error while decoding MB 46 26, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 494, "h264_decode_error_count": 1440, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:42:32+09:00", "last_error_summary": "[h264 @ 0x559c493537c0] error while decoding MB 46 26, bytestream -5", "last_exit_at": "2026-07-01T10:42:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 98.518, "last_started_at": "2026-07-01T10:40:53+09:00", "pid": 97672, "process": {"exists": false, "pid": 97672}, "restart_count": 541, "restart_count_per_hour": 22, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 542}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.905, "sample_count": 1369, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448003, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:42:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T10:42:32+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-01T10:42:32+09:00", "mjpeg": {"age_sec": 0.152, "exists": true, "mtime": "2026-07-01T10:42:32.157289+09:00", "mtime_epoch": 1782870152.1572886, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:42:32+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5056.266, "name": "celery", "pid": 267, "rss_mb": 5056.266, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 48, "pushed_frames": 48, "read_frames": 47, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.6, "pushed": 1.6, "read": 1.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.189, "count": 47, "max_ms": 53.003, "p95_ms": 35.999}, "gstreamer_push": {"avg_ms": 1.03, "count": 48, "max_ms": 1.212, "p95_ms": 1.136}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.409, "count": 47, "max_ms": 41.904, "p95_ms": 39.468}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.548, "count": 47, "max_ms": 87.996, "p95_ms": 81.075}, "yolo11_640_fire_11class": {"avg_ms": 47.492, "count": 47, "max_ms": 66.044, "p95_ms": 62.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.289, "count": 47, "max_ms": 83.044, "p95_ms": 77.579}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.666, "count": 48, "max_ms": 185.81, "p95_ms": 174.905}, "read_interval": {"avg_ms": 638.376, "count": 47, "max_ms": 14775.155, "p95_ms": 527.694}, "triton_wait": {"avg_ms": 62.584, "count": 47, "max_ms": 88.008, "p95_ms": 82.099}, "visualize": {"avg_ms": 5.032, "count": 47, "max_ms": 6.423, "p95_ms": 5.537}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12066}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10986.707, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 494, "h264_decode_error_count": 1440, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:42:32+09:00", "last_error_summary": "[h264 @ 0x559c493537c0] error while decoding MB 46 26, bytestream -5", "last_exit_at": "2026-07-01T10:42:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 98.522, "last_started_at": "2026-07-01T10:40:53+09:00", "pid": 97672, "process": {"exists": false, "pid": 97672}, "restart_count": 541, "restart_count_per_hour": 22, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 542}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.905, "sample_count": 1369, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448003, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:42:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T10:42:32+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-01T10:42:32+09:00", "mjpeg": {"age_sec": 0.156, "exists": true, "mtime": "2026-07-01T10:42:32.157289+09:00", "mtime_epoch": 1782870152.1572886, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:42:32+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5056.266, "name": "celery", "pid": 267, "rss_mb": 5056.266, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 47, "pushed_frames": 47, "read_frames": 47, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.567, "pushed": 1.567, "read": 1.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.189, "count": 47, "max_ms": 53.003, "p95_ms": 35.999}, "gstreamer_push": {"avg_ms": 1.028, "count": 47, "max_ms": 1.212, "p95_ms": 1.136}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.409, "count": 47, "max_ms": 41.904, "p95_ms": 39.468}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.548, "count": 47, "max_ms": 87.996, "p95_ms": 81.075}, "yolo11_640_fire_11class": {"avg_ms": 47.492, "count": 47, "max_ms": 66.044, "p95_ms": 62.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.289, "count": 47, "max_ms": 83.044, "p95_ms": 77.579}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.459, "count": 47, "max_ms": 185.81, "p95_ms": 174.905}, "read_interval": {"avg_ms": 638.376, "count": 47, "max_ms": 14775.155, "p95_ms": 527.694}, "triton_wait": {"avg_ms": 62.584, "count": 47, "max_ms": 88.008, "p95_ms": 82.099}, "visualize": {"avg_ms": 5.032, "count": 47, "max_ms": 6.423, "p95_ms": 5.537}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12066}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10997.848, "limit_mb": 8796093022207.996}, "detail": {"pid": 97931, "restart_count": 542, "start_count": 543}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 494, "h264_decode_error_count": 1440, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:42:32+09:00", "last_error_summary": "[h264 @ 0x559c493537c0] error while decoding MB 46 26, bytestream -5", "last_exit_at": "2026-07-01T10:42:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:42:35+09:00", "pid": 97931, "process": {"exists": true, "fd_count": 4, "hwm_mb": 46.684, "name": "ffmpeg", "pid": 97931, "rss_mb": 46.684, "state": "S (sleeping)", "threads": 1}, "restart_count": 542, "restart_count_per_hour": 23, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 543}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.905, "sample_count": 1341, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448003, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:42:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.779, "last_frame_pushed_at": "2026-07-01T10:42:32+09:00", "last_frame_read_age_sec": 4.935, "last_frame_read_at": "2026-07-01T10:42:32+09:00", "mjpeg": {"age_sec": 4.76, "exists": true, "mtime": "2026-07-01T10:42:32.325292+09:00", "mtime_epoch": 1782870152.3252919, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174361}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:42:37+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5056.266, "name": "celery", "pid": 267, "rss_mb": 5056.266, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 26, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.565, "pushed": 1.565, "read": 1.565}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.147, "count": 26, "max_ms": 53.003, "p95_ms": 48.043}, "gstreamer_push": {"avg_ms": 1.024, "count": 26, "max_ms": 1.151, "p95_ms": 1.136}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.336, "count": 26, "max_ms": 40.722, "p95_ms": 39.468}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.926, "count": 26, "max_ms": 81.075, "p95_ms": 81.047}, "yolo11_640_fire_11class": {"avg_ms": 48.204, "count": 26, "max_ms": 66.044, "p95_ms": 62.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.03, "count": 26, "max_ms": 82.15, "p95_ms": 74.443}}, "truncated_count": 0}, "processing_total": {"avg_ms": 149.957, "count": 26, "max_ms": 185.81, "p95_ms": 174.04}, "read_interval": {"avg_ms": 1017.607, "count": 26, "max_ms": 14775.155, "p95_ms": 7384.06}, "triton_wait": {"avg_ms": 61.981, "count": 26, "max_ms": 82.162, "p95_ms": 81.087}, "visualize": {"avg_ms": 4.953, "count": 26, "max_ms": 6.423, "p95_ms": 5.709}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.618, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12068}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10988.277, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560e03e3f940] error while decoding MB 75 31, bytestream -7 [in#0/rtsp @ 0x560e03d11e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x560e03d11e00] Error retrieving a packet from demuxer: Input/output error", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 495, "h264_decode_error_count": 1441, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:43:55+09:00", "last_error_summary": "[h264 @ 0x560e03e3f940] error while decoding MB 75 31, bytestream -7 [in#0/rtsp @ 0x560e03d11e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x560e03d11e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:42:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 80.859, "last_started_at": "2026-07-01T10:42:35+09:00", "pid": 97931, "process": {"exists": false, "pid": 97931}, "restart_count": 542, "restart_count_per_hour": 23, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 543}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.781, "sample_count": 1287, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448260, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:43:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T10:43:55+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T10:43:55+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T10:43:55.898837+09:00", "mtime_epoch": 1782870235.8988366, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172268}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:43:55+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5056.266, "name": "celery", "pid": 267, "rss_mb": 5056.266, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.416, "pushed": 0.416, "read": 0.45}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.069, "count": 12, "max_ms": 41.108, "p95_ms": 41.108}, "gstreamer_push": {"avg_ms": 1.095, "count": 12, "max_ms": 1.302, "p95_ms": 1.302}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.495, "count": 12, "max_ms": 55.398, "p95_ms": 55.398}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.066, "count": 12, "max_ms": 79.805, "p95_ms": 79.805}, "yolo11_640_fire_11class": {"avg_ms": 44.473, "count": 12, "max_ms": 61.58, "p95_ms": 61.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.259, "count": 12, "max_ms": 64.467, "p95_ms": 64.467}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.994, "count": 12, "max_ms": 155.282, "p95_ms": 155.282}, "read_interval": {"avg_ms": 2627.81, "count": 13, "max_ms": 17442.027, "p95_ms": 17442.027}, "triton_wait": {"avg_ms": 60.077, "count": 12, "max_ms": 79.816, "p95_ms": 79.816}, "visualize": {"avg_ms": 4.59, "count": 12, "max_ms": 6.785, "p95_ms": 6.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.859, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12068}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10987.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:43:55+09:00", "last_error_summary": "[h264 @ 0x560e03e3f940] error while decoding MB 75 31, bytestream -7 [in#0/rtsp @ 0x560e03d11e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x560e03d11e00] Error retrieving a packet from demuxer: Input/output error", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 495, "h264_decode_error_count": 1441, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:43:55+09:00", "last_error_summary": "[h264 @ 0x560e03e3f940] error while decoding MB 75 31, bytestream -7 [in#0/rtsp @ 0x560e03d11e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x560e03d11e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:43:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 80.946, "last_started_at": "2026-07-01T10:42:35+09:00", "pid": 97931, "process": {"exists": false, "pid": 97931}, "restart_count": 542, "restart_count_per_hour": 23, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 543}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.781, "sample_count": 1288, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448261, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:43:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T10:43:56+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-01T10:43:55+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-01T10:43:55.898837+09:00", "mtime_epoch": 1782870235.8988366, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172268}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:43:56+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5056.266, "name": "celery", "pid": 267, "rss_mb": 5056.266, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.449, "pushed": 0.449, "read": 0.449}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.032, "count": 13, "max_ms": 41.108, "p95_ms": 41.108}, "gstreamer_push": {"avg_ms": 1.096, "count": 13, "max_ms": 1.302, "p95_ms": 1.302}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.386, "count": 13, "max_ms": 55.398, "p95_ms": 55.398}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.232, "count": 13, "max_ms": 79.805, "p95_ms": 79.805}, "yolo11_640_fire_11class": {"avg_ms": 44.867, "count": 13, "max_ms": 61.58, "p95_ms": 61.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.769, "count": 13, "max_ms": 64.467, "p95_ms": 64.467}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.948, "count": 13, "max_ms": 155.282, "p95_ms": 155.282}, "read_interval": {"avg_ms": 2627.81, "count": 13, "max_ms": 17442.027, "p95_ms": 17442.027}, "triton_wait": {"avg_ms": 60.244, "count": 13, "max_ms": 79.816, "p95_ms": 79.816}, "visualize": {"avg_ms": 4.573, "count": 13, "max_ms": 6.785, "p95_ms": 6.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.946, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12068}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10987.363, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 495, "h264_decode_error_count": 1441, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:43:55+09:00", "last_error_summary": "[h264 @ 0x560e03e3f940] error while decoding MB 75 31, bytestream -7 [in#0/rtsp @ 0x560e03d11e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x560e03d11e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:43:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 80.95, "last_started_at": "2026-07-01T10:42:35+09:00", "pid": 97931, "process": {"exists": false, "pid": 97931}, "restart_count": 542, "restart_count_per_hour": 23, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 543}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.781, "sample_count": 1288, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448261, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:43:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T10:43:56+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T10:43:55+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T10:43:55.898837+09:00", "mtime_epoch": 1782870235.8988366, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172268}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:43:56+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5056.266, "name": "celery", "pid": 267, "rss_mb": 5056.266, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.449, "pushed": 0.449, "read": 0.449}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.032, "count": 13, "max_ms": 41.108, "p95_ms": 41.108}, "gstreamer_push": {"avg_ms": 1.096, "count": 13, "max_ms": 1.302, "p95_ms": 1.302}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.386, "count": 13, "max_ms": 55.398, "p95_ms": 55.398}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.232, "count": 13, "max_ms": 79.805, "p95_ms": 79.805}, "yolo11_640_fire_11class": {"avg_ms": 44.867, "count": 13, "max_ms": 61.58, "p95_ms": 61.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.769, "count": 13, "max_ms": 64.467, "p95_ms": 64.467}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.948, "count": 13, "max_ms": 155.282, "p95_ms": 155.282}, "read_interval": {"avg_ms": 2627.81, "count": 13, "max_ms": 17442.027, "p95_ms": 17442.027}, "triton_wait": {"avg_ms": 60.244, "count": 13, "max_ms": 79.816, "p95_ms": 79.816}, "visualize": {"avg_ms": 4.573, "count": 13, "max_ms": 6.785, "p95_ms": 6.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.95, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12068}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11092.465, "limit_mb": 8796093022207.996}, "detail": {"pid": 98038, "restart_count": 543, "start_count": 544}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 495, "h264_decode_error_count": 1441, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:43:55+09:00", "last_error_summary": "[h264 @ 0x560e03e3f940] error while decoding MB 75 31, bytestream -7 [in#0/rtsp @ 0x560e03d11e00] Error during demuxing: Input/output error [in#0/rtsp @ 0x560e03d11e00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:43:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:43:58+09:00", "pid": 98038, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.086, "name": "ffmpeg", "pid": 98038, "rss_mb": 136.086, "state": "S (sleeping)", "threads": 106}, "restart_count": 543, "restart_count_per_hour": 24, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 544}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.839, "sample_count": 1264, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448261, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:43:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.641, "last_frame_pushed_at": "2026-07-01T10:43:56+09:00", "last_frame_read_age_sec": 4.776, "last_frame_read_at": "2026-07-01T10:43:55+09:00", "mjpeg": {"age_sec": 4.621, "exists": true, "mtime": "2026-07-01T10:43:56.046839+09:00", "mtime_epoch": 1782870236.0468392, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:44:00+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5056.266, "name": "celery", "pid": 267, "rss_mb": 5056.266, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 10, "pushed_frames": 10, "read_frames": 10, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.427, "pushed": 0.427, "read": 0.427}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.837, "count": 10, "max_ms": 34.464, "p95_ms": 34.464}, "gstreamer_push": {"avg_ms": 1.079, "count": 10, "max_ms": 1.148, "p95_ms": 1.148}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.107, "count": 10, "max_ms": 55.398, "p95_ms": 55.398}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.166, "count": 10, "max_ms": 79.805, "p95_ms": 79.805}, "yolo11_640_fire_11class": {"avg_ms": 47.759, "count": 10, "max_ms": 61.58, "p95_ms": 61.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.222, "count": 10, "max_ms": 64.467, "p95_ms": 64.467}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.576, "count": 10, "max_ms": 155.282, "p95_ms": 155.282}, "read_interval": {"avg_ms": 2852.39, "count": 10, "max_ms": 17442.027, "p95_ms": 17442.027}, "triton_wait": {"avg_ms": 62.178, "count": 10, "max_ms": 79.816, "p95_ms": 79.816}, "visualize": {"avg_ms": 4.577, "count": 10, "max_ms": 6.785, "p95_ms": 6.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.402, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12074}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10991.004, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556b72be1640] error while decoding MB 13 24, bytestream -11 [in#0/rtsp @ 0x556b72229e00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x556b72229e00] Error retrieving a packet from demuxer: Connection reset by peer", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 496, "h264_decode_error_count": 1442, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:46:10+09:00", "last_error_summary": "[h264 @ 0x556b72be1640] error while decoding MB 13 24, bytestream -11 [in#0/rtsp @ 0x556b72229e00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x556b72229e00] Error retrieving a packet from demuxer: Connection reset by peer", "last_exit_at": "2026-07-01T10:43:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.291, "last_started_at": "2026-07-01T10:43:58+09:00", "pid": 98038, "process": {"exists": false, "pid": 98038}, "restart_count": 543, "restart_count_per_hour": 23, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 544}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.243, "sample_count": 1243, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448897, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:46:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T10:46:10+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T10:46:10+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T10:46:10.913310+09:00", "mtime_epoch": 1782870370.9133096, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:46:10+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5056.781, "name": "celery", "pid": 267, "rss_mb": 5056.781, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 10, "pushed_frames": 10, "read_frames": 11, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.361, "pushed": 0.361, "read": 0.397}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.68, "count": 10, "max_ms": 36.59, "p95_ms": 36.59}, "gstreamer_push": {"avg_ms": 1.129, "count": 10, "max_ms": 1.272, "p95_ms": 1.272}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.637, "count": 10, "max_ms": 33.509, "p95_ms": 33.509}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.13, "count": 10, "max_ms": 83.196, "p95_ms": 83.196}, "yolo11_640_fire_11class": {"avg_ms": 47.246, "count": 10, "max_ms": 62.978, "p95_ms": 62.978}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.165, "count": 10, "max_ms": 79.17, "p95_ms": 79.17}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.438, "count": 10, "max_ms": 152.349, "p95_ms": 152.349}, "read_interval": {"avg_ms": 3207.318, "count": 11, "max_ms": 19138.014, "p95_ms": 19138.014}, "triton_wait": {"avg_ms": 65.143, "count": 10, "max_ms": 83.208, "p95_ms": 83.208}, "visualize": {"avg_ms": 4.199, "count": 10, "max_ms": 4.515, "p95_ms": 4.515}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.697, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12074}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10990.098, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:46:10+09:00", "last_error_summary": "[h264 @ 0x556b72be1640] error while decoding MB 13 24, bytestream -11 [in#0/rtsp @ 0x556b72229e00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x556b72229e00] Error retrieving a packet from demuxer: Connection reset by peer", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 496, "h264_decode_error_count": 1442, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:46:10+09:00", "last_error_summary": "[h264 @ 0x556b72be1640] error while decoding MB 13 24, bytestream -11 [in#0/rtsp @ 0x556b72229e00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x556b72229e00] Error retrieving a packet from demuxer: Connection reset by peer", "last_exit_at": "2026-07-01T10:46:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.381, "last_started_at": "2026-07-01T10:43:58+09:00", "pid": 98038, "process": {"exists": false, "pid": 98038}, "restart_count": 543, "restart_count_per_hour": 23, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 544}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.109, "sample_count": 1243, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448898, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:46:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T10:46:11+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T10:46:10+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T10:46:10.913310+09:00", "mtime_epoch": 1782870370.9133096, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:46:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5056.781, "name": "celery", "pid": 267, "rss_mb": 5056.781, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 11, "pushed_frames": 11, "read_frames": 11, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.396, "pushed": 0.396, "read": 0.396}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.815, "count": 11, "max_ms": 36.59, "p95_ms": 36.59}, "gstreamer_push": {"avg_ms": 1.126, "count": 11, "max_ms": 1.272, "p95_ms": 1.272}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.755, "count": 11, "max_ms": 33.509, "p95_ms": 33.509}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.169, "count": 11, "max_ms": 83.196, "p95_ms": 83.196}, "yolo11_640_fire_11class": {"avg_ms": 47.813, "count": 11, "max_ms": 62.978, "p95_ms": 62.978}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.439, "count": 11, "max_ms": 79.17, "p95_ms": 79.17}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.455, "count": 11, "max_ms": 152.349, "p95_ms": 152.349}, "read_interval": {"avg_ms": 3207.318, "count": 11, "max_ms": 19138.014, "p95_ms": 19138.014}, "triton_wait": {"avg_ms": 65.182, "count": 11, "max_ms": 83.208, "p95_ms": 83.208}, "visualize": {"avg_ms": 4.175, "count": 11, "max_ms": 4.515, "p95_ms": 4.515}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.787, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12074}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10990.215, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 496, "h264_decode_error_count": 1442, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:46:10+09:00", "last_error_summary": "[h264 @ 0x556b72be1640] error while decoding MB 13 24, bytestream -11 [in#0/rtsp @ 0x556b72229e00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x556b72229e00] Error retrieving a packet from demuxer: Connection reset by peer", "last_exit_at": "2026-07-01T10:46:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.385, "last_started_at": "2026-07-01T10:43:58+09:00", "pid": 98038, "process": {"exists": false, "pid": 98038}, "restart_count": 543, "restart_count_per_hour": 23, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 544}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.109, "sample_count": 1243, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448898, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:46:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T10:46:11+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-01T10:46:10+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-01T10:46:10.913310+09:00", "mtime_epoch": 1782870370.9133096, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169054}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:46:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5056.781, "name": "celery", "pid": 267, "rss_mb": 5056.781, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 11, "pushed_frames": 11, "read_frames": 11, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.396, "pushed": 0.396, "read": 0.396}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.815, "count": 11, "max_ms": 36.59, "p95_ms": 36.59}, "gstreamer_push": {"avg_ms": 1.126, "count": 11, "max_ms": 1.272, "p95_ms": 1.272}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.755, "count": 11, "max_ms": 33.509, "p95_ms": 33.509}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.169, "count": 11, "max_ms": 83.196, "p95_ms": 83.196}, "yolo11_640_fire_11class": {"avg_ms": 47.813, "count": 11, "max_ms": 62.978, "p95_ms": 62.978}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.439, "count": 11, "max_ms": 79.17, "p95_ms": 79.17}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.455, "count": 11, "max_ms": 152.349, "p95_ms": 152.349}, "read_interval": {"avg_ms": 3207.318, "count": 11, "max_ms": 19138.014, "p95_ms": 19138.014}, "triton_wait": {"avg_ms": 65.182, "count": 11, "max_ms": 83.208, "p95_ms": 83.208}, "visualize": {"avg_ms": 4.175, "count": 11, "max_ms": 4.515, "p95_ms": 4.515}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.791, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12074}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11093.078, "limit_mb": 8796093022207.996}, "detail": {"pid": 98145, "restart_count": 544, "start_count": 545}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 496, "h264_decode_error_count": 1442, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:46:10+09:00", "last_error_summary": "[h264 @ 0x556b72be1640] error while decoding MB 13 24, bytestream -11 [in#0/rtsp @ 0x556b72229e00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x556b72229e00] Error retrieving a packet from demuxer: Connection reset by peer", "last_exit_at": "2026-07-01T10:46:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:46:13+09:00", "pid": 98145, "process": {"exists": true, "fd_count": 5, "hwm_mb": 134.125, "name": "ffmpeg", "pid": 98145, "rss_mb": 134.125, "state": "S (sleeping)", "threads": 106}, "restart_count": 544, "restart_count_per_hour": 24, "rtsp_timeout_count": 410, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 545}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.395, "sample_count": 1216, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1448898, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:46:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.458, "last_frame_pushed_at": "2026-07-01T10:46:11+09:00", "last_frame_read_age_sec": 4.595, "last_frame_read_at": "2026-07-01T10:46:10+09:00", "mjpeg": {"age_sec": 4.441, "exists": true, "mtime": "2026-07-01T10:46:11.061312+09:00", "mtime_epoch": 1782870371.0613122, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169808}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:46:15+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5056.781, "name": "celery", "pid": 267, "rss_mb": 5056.781, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 9, "pushed_frames": 9, "read_frames": 9, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.695, "pushed": 0.695, "read": 0.695}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.361, "count": 9, "max_ms": 33.164, "p95_ms": 33.164}, "gstreamer_push": {"avg_ms": 1.125, "count": 9, "max_ms": 1.272, "p95_ms": 1.272}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.772, "count": 9, "max_ms": 33.509, "p95_ms": 33.509}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.205, "count": 9, "max_ms": 83.196, "p95_ms": 83.196}, "yolo11_640_fire_11class": {"avg_ms": 48.657, "count": 9, "max_ms": 62.978, "p95_ms": 62.978}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.0, "count": 9, "max_ms": 79.17, "p95_ms": 79.17}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.991, "count": 9, "max_ms": 152.349, "p95_ms": 152.349}, "read_interval": {"avg_ms": 3055.28, "count": 9, "max_ms": 19138.014, "p95_ms": 19138.014}, "triton_wait": {"avg_ms": 64.217, "count": 9, "max_ms": 83.208, "p95_ms": 83.208}, "visualize": {"avg_ms": 4.185, "count": 9, "max_ms": 4.515, "p95_ms": 4.515}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 12.955, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12076}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10991.512, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ff78cc1e40] error while decoding MB 29 31, bytestream -7 [in#0/rtsp @ 0x55ff78b75e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff78b75e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 497, "h264_decode_error_count": 1443, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:48:22+09:00", "last_error_summary": "[h264 @ 0x55ff78cc1e40] error while decoding MB 29 31, bytestream -7 [in#0/rtsp @ 0x55ff78b75e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff78b75e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:46:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.522, "last_started_at": "2026-07-01T10:46:13+09:00", "pid": 98145, "process": {"exists": false, "pid": 98145}, "restart_count": 544, "restart_count_per_hour": 23, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 545}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.354, "sample_count": 926, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449174, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:48:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.053, "last_frame_pushed_at": "2026-07-01T10:48:21+09:00", "last_frame_read_age_sec": 0.044, "last_frame_read_at": "2026-07-01T10:48:21+09:00", "mjpeg": {"age_sec": 0.032, "exists": true, "mtime": "2026-07-01T10:48:21.991689+09:00", "mtime_epoch": 1782870501.9916892, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171053}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:48:22+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 16, "pushed_frames": 16, "read_frames": 17, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.537, "pushed": 0.537, "read": 0.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.913, "count": 16, "max_ms": 35.038, "p95_ms": 35.038}, "gstreamer_push": {"avg_ms": 1.089, "count": 16, "max_ms": 1.183, "p95_ms": 1.183}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.934, "count": 16, "max_ms": 46.037, "p95_ms": 46.037}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.235, "count": 16, "max_ms": 88.275, "p95_ms": 88.275}, "yolo11_640_fire_11class": {"avg_ms": 46.727, "count": 16, "max_ms": 62.641, "p95_ms": 62.641}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.823, "count": 16, "max_ms": 80.813, "p95_ms": 80.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.447, "count": 16, "max_ms": 160.37, "p95_ms": 160.37}, "read_interval": {"avg_ms": 2289.49, "count": 17, "max_ms": 27309.058, "p95_ms": 27309.058}, "triton_wait": {"avg_ms": 65.247, "count": 16, "max_ms": 88.286, "p95_ms": 88.286}, "visualize": {"avg_ms": 4.411, "count": 16, "max_ms": 4.824, "p95_ms": 4.824}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.801, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12076}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10990.848, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:48:22+09:00", "last_error_summary": "[h264 @ 0x55ff78cc1e40] error while decoding MB 29 31, bytestream -7 [in#0/rtsp @ 0x55ff78b75e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff78b75e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 497, "h264_decode_error_count": 1443, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:48:22+09:00", "last_error_summary": "[h264 @ 0x55ff78cc1e40] error while decoding MB 29 31, bytestream -7 [in#0/rtsp @ 0x55ff78b75e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff78b75e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:48:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.62, "last_started_at": "2026-07-01T10:46:13+09:00", "pid": 98145, "process": {"exists": false, "pid": 98145}, "restart_count": 544, "restart_count_per_hour": 23, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 545}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.354, "sample_count": 927, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449175, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:48:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T10:48:22+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T10:48:21+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T10:48:21.991689+09:00", "mtime_epoch": 1782870501.9916892, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171053}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:48:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 17, "pushed_frames": 17, "read_frames": 17, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.569, "pushed": 0.569, "read": 0.569}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.95, "count": 17, "max_ms": 35.038, "p95_ms": 35.038}, "gstreamer_push": {"avg_ms": 1.088, "count": 17, "max_ms": 1.183, "p95_ms": 1.183}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.805, "count": 17, "max_ms": 46.037, "p95_ms": 46.037}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.39, "count": 17, "max_ms": 88.275, "p95_ms": 88.275}, "yolo11_640_fire_11class": {"avg_ms": 47.029, "count": 17, "max_ms": 62.641, "p95_ms": 62.641}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.063, "count": 17, "max_ms": 80.813, "p95_ms": 80.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.475, "count": 17, "max_ms": 160.37, "p95_ms": 160.37}, "read_interval": {"avg_ms": 2289.49, "count": 17, "max_ms": 27309.058, "p95_ms": 27309.058}, "triton_wait": {"avg_ms": 65.403, "count": 17, "max_ms": 88.286, "p95_ms": 88.286}, "visualize": {"avg_ms": 4.399, "count": 17, "max_ms": 4.824, "p95_ms": 4.824}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12076}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10990.965, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 497, "h264_decode_error_count": 1443, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:48:22+09:00", "last_error_summary": "[h264 @ 0x55ff78cc1e40] error while decoding MB 29 31, bytestream -7 [in#0/rtsp @ 0x55ff78b75e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff78b75e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:48:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.624, "last_started_at": "2026-07-01T10:46:13+09:00", "pid": 98145, "process": {"exists": false, "pid": 98145}, "restart_count": 544, "restart_count_per_hour": 23, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 545}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.354, "sample_count": 927, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449175, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:48:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T10:48:22+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-01T10:48:21+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-01T10:48:21.991689+09:00", "mtime_epoch": 1782870501.9916892, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171053}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:48:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 17, "pushed_frames": 17, "read_frames": 17, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.569, "pushed": 0.569, "read": 0.569}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.95, "count": 17, "max_ms": 35.038, "p95_ms": 35.038}, "gstreamer_push": {"avg_ms": 1.088, "count": 17, "max_ms": 1.183, "p95_ms": 1.183}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.805, "count": 17, "max_ms": 46.037, "p95_ms": 46.037}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.39, "count": 17, "max_ms": 88.275, "p95_ms": 88.275}, "yolo11_640_fire_11class": {"avg_ms": 47.029, "count": 17, "max_ms": 62.641, "p95_ms": 62.641}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.063, "count": 17, "max_ms": 80.813, "p95_ms": 80.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.475, "count": 17, "max_ms": 160.37, "p95_ms": 160.37}, "read_interval": {"avg_ms": 2289.49, "count": 17, "max_ms": 27309.058, "p95_ms": 27309.058}, "triton_wait": {"avg_ms": 65.403, "count": 17, "max_ms": 88.286, "p95_ms": 88.286}, "visualize": {"avg_ms": 4.399, "count": 17, "max_ms": 4.824, "p95_ms": 4.824}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12076}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11001.547, "limit_mb": 8796093022207.996}, "detail": {"pid": 98252, "restart_count": 545, "start_count": 546}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 497, "h264_decode_error_count": 1443, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:48:22+09:00", "last_error_summary": "[h264 @ 0x55ff78cc1e40] error while decoding MB 29 31, bytestream -7 [in#0/rtsp @ 0x55ff78b75e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff78b75e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:48:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:48:24+09:00", "pid": 98252, "process": {"exists": true, "fd_count": 4, "hwm_mb": 46.605, "name": "ffmpeg", "pid": 98252, "rss_mb": 46.605, "state": "S (sleeping)", "threads": 1}, "restart_count": 545, "restart_count_per_hour": 24, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 546}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.354, "sample_count": 927, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449175, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:48:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.335, "last_frame_pushed_at": "2026-07-01T10:48:22+09:00", "last_frame_read_age_sec": 4.474, "last_frame_read_at": "2026-07-01T10:48:21+09:00", "mjpeg": {"age_sec": 4.318, "exists": true, "mtime": "2026-07-01T10:48:22.135692+09:00", "mtime_epoch": 1782870502.1356916, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171051}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:48:26+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 9, "pushed_frames": 9, "read_frames": 9, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.594, "pushed": 1.594, "read": 1.594}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 31.595, "count": 9, "max_ms": 35.038, "p95_ms": 35.038}, "gstreamer_push": {"avg_ms": 1.08, "count": 9, "max_ms": 1.183, "p95_ms": 1.183}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.021, "count": 9, "max_ms": 31.852, "p95_ms": 31.852}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.791, "count": 9, "max_ms": 82.388, "p95_ms": 82.388}, "yolo11_640_fire_11class": {"avg_ms": 45.212, "count": 9, "max_ms": 57.254, "p95_ms": 57.254}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.048, "count": 9, "max_ms": 75.524, "p95_ms": 75.524}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.997, "count": 9, "max_ms": 155.354, "p95_ms": 155.354}, "read_interval": {"avg_ms": 3164.63, "count": 9, "max_ms": 27309.058, "p95_ms": 27309.058}, "triton_wait": {"avg_ms": 64.803, "count": 9, "max_ms": 82.4, "p95_ms": 82.4}, "visualize": {"avg_ms": 4.444, "count": 9, "max_ms": 4.824, "p95_ms": 4.824}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.647, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10992.316, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a1a842db40] error while decoding MB 66 25, bytestream -44", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 498, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:49:33+09:00", "last_error_summary": "[h264 @ 0x55a1a842db40] error while decoding MB 66 25, bytestream -44", "last_exit_at": "2026-07-01T10:48:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 68.746, "last_started_at": "2026-07-01T10:48:24+09:00", "pid": 98252, "process": {"exists": false, "pid": 98252}, "restart_count": 545, "restart_count_per_hour": 23, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 546}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.55, "sample_count": 798, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449263, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:49:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.048, "last_frame_pushed_at": "2026-07-01T10:49:33+09:00", "last_frame_read_age_sec": 0.041, "last_frame_read_at": "2026-07-01T10:49:33+09:00", "mjpeg": {"age_sec": 0.034, "exists": true, "mtime": "2026-07-01T10:49:33.164974+09:00", "mtime_epoch": 1782870573.1649742, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169494}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:49:33+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 60, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.253, "pushed": 2.253, "read": 2.291}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.908, "count": 59, "max_ms": 43.348, "p95_ms": 35.51}, "gstreamer_push": {"avg_ms": 1.032, "count": 59, "max_ms": 1.232, "p95_ms": 1.154}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.43, "count": 59, "max_ms": 40.278, "p95_ms": 38.06}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.668, "count": 59, "max_ms": 90.399, "p95_ms": 87.297}, "yolo11_640_fire_11class": {"avg_ms": 48.798, "count": 59, "max_ms": 65.612, "p95_ms": 63.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.259, "count": 59, "max_ms": 82.506, "p95_ms": 80.903}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.997, "count": 59, "max_ms": 191.734, "p95_ms": 183.324}, "read_interval": {"avg_ms": 588.675, "count": 60, "max_ms": 9175.273, "p95_ms": 3656.04}, "triton_wait": {"avg_ms": 66.681, "count": 59, "max_ms": 90.412, "p95_ms": 87.309}, "visualize": {"avg_ms": 4.951, "count": 59, "max_ms": 7.054, "p95_ms": 6.032}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 26.186, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10991.031, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:49:33+09:00", "last_error_summary": "[h264 @ 0x55a1a842db40] error while decoding MB 66 25, bytestream -44", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 498, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:49:33+09:00", "last_error_summary": "[h264 @ 0x55a1a842db40] error while decoding MB 66 25, bytestream -44", "last_exit_at": "2026-07-01T10:49:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 68.86, "last_started_at": "2026-07-01T10:48:24+09:00", "pid": 98252, "process": {"exists": false, "pid": 98252}, "restart_count": 545, "restart_count_per_hour": 23, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 546}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.55, "sample_count": 798, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:49:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T10:49:33+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-01T10:49:33+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-01T10:49:33.164974+09:00", "mtime_epoch": 1782870573.1649742, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169494}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:49:33+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 60, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.281, "pushed": 2.281, "read": 2.281}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.01, "count": 60, "max_ms": 43.348, "p95_ms": 35.51}, "gstreamer_push": {"avg_ms": 1.03, "count": 60, "max_ms": 1.232, "p95_ms": 1.14}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.408, "count": 60, "max_ms": 40.278, "p95_ms": 37.521}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.503, "count": 60, "max_ms": 90.399, "p95_ms": 86.865}, "yolo11_640_fire_11class": {"avg_ms": 48.7, "count": 60, "max_ms": 65.612, "p95_ms": 61.879}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.158, "count": 60, "max_ms": 82.506, "p95_ms": 77.414}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.922, "count": 60, "max_ms": 191.734, "p95_ms": 183.188}, "read_interval": {"avg_ms": 588.675, "count": 60, "max_ms": 9175.273, "p95_ms": 3656.04}, "triton_wait": {"avg_ms": 66.515, "count": 60, "max_ms": 90.412, "p95_ms": 86.878}, "visualize": {"avg_ms": 4.934, "count": 60, "max_ms": 7.054, "p95_ms": 5.827}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 26.3, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10991.148, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 498, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:49:33+09:00", "last_error_summary": "[h264 @ 0x55a1a842db40] error while decoding MB 66 25, bytestream -44", "last_exit_at": "2026-07-01T10:49:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 68.866, "last_started_at": "2026-07-01T10:48:24+09:00", "pid": 98252, "process": {"exists": false, "pid": 98252}, "restart_count": 545, "restart_count_per_hour": 23, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 546}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.55, "sample_count": 798, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:49:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T10:49:33+09:00", "last_frame_read_age_sec": 0.161, "last_frame_read_at": "2026-07-01T10:49:33+09:00", "mjpeg": {"age_sec": 0.154, "exists": true, "mtime": "2026-07-01T10:49:33.164974+09:00", "mtime_epoch": 1782870573.1649742, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169494}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:49:33+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 60, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.281, "pushed": 2.281, "read": 2.281}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.01, "count": 60, "max_ms": 43.348, "p95_ms": 35.51}, "gstreamer_push": {"avg_ms": 1.03, "count": 60, "max_ms": 1.232, "p95_ms": 1.14}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.408, "count": 60, "max_ms": 40.278, "p95_ms": 37.521}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.503, "count": 60, "max_ms": 90.399, "p95_ms": 86.865}, "yolo11_640_fire_11class": {"avg_ms": 48.7, "count": 60, "max_ms": 65.612, "p95_ms": 61.879}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.158, "count": 60, "max_ms": 82.506, "p95_ms": 77.414}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.922, "count": 60, "max_ms": 191.734, "p95_ms": 183.188}, "read_interval": {"avg_ms": 588.675, "count": 60, "max_ms": 9175.273, "p95_ms": 3656.04}, "triton_wait": {"avg_ms": 66.515, "count": 60, "max_ms": 90.412, "p95_ms": 86.878}, "visualize": {"avg_ms": 4.934, "count": 60, "max_ms": 7.054, "p95_ms": 5.827}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 26.306, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11000.809, "limit_mb": 8796093022207.996}, "detail": {"pid": 98359, "restart_count": 546, "start_count": 547}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 498, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:49:33+09:00", "last_error_summary": "[h264 @ 0x55a1a842db40] error while decoding MB 66 25, bytestream -44", "last_exit_at": "2026-07-01T10:49:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:49:35+09:00", "pid": 98359, "process": {"exists": true, "fd_count": 4, "hwm_mb": 44.133, "name": "ffmpeg", "pid": 98359, "rss_mb": 44.133, "state": "S (sleeping)", "threads": 1}, "restart_count": 546, "restart_count_per_hour": 24, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 547}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.654, "sample_count": 768, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:49:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.554, "last_frame_pushed_at": "2026-07-01T10:49:33+09:00", "last_frame_read_age_sec": 4.705, "last_frame_read_at": "2026-07-01T10:49:33+09:00", "mjpeg": {"age_sec": 4.538, "exists": true, "mtime": "2026-07-01T10:49:33.324977+09:00", "mtime_epoch": 1782870573.324977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169815}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:49:37+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 56, "select_timeouts": 13, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.906, "pushed": 1.906, "read": 1.873}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.945, "count": 57, "max_ms": 43.348, "p95_ms": 36.722}, "gstreamer_push": {"avg_ms": 1.031, "count": 57, "max_ms": 1.232, "p95_ms": 1.154}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.311, "count": 57, "max_ms": 40.278, "p95_ms": 37.521}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.56, "count": 57, "max_ms": 90.399, "p95_ms": 87.297}, "yolo11_640_fire_11class": {"avg_ms": 48.915, "count": 57, "max_ms": 65.612, "p95_ms": 63.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.437, "count": 57, "max_ms": 82.506, "p95_ms": 80.903}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.009, "count": 57, "max_ms": 191.734, "p95_ms": 183.324}, "read_interval": {"avg_ms": 452.787, "count": 56, "max_ms": 7402.747, "p95_ms": 3656.04}, "triton_wait": {"avg_ms": 66.572, "count": 57, "max_ms": 90.412, "p95_ms": 87.309}, "visualize": {"avg_ms": 4.949, "count": 57, "max_ms": 7.054, "p95_ms": 6.032}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11002.184, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 498, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:49:33+09:00", "last_error_summary": "[h264 @ 0x55a1a842db40] error while decoding MB 66 25, bytestream -44", "last_exit_at": "2026-07-01T10:49:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 32.065, "last_started_at": "2026-07-01T10:49:35+09:00", "pid": 98359, "process": {"exists": true, "fd_count": 4, "hwm_mb": 47.098, "name": "ffmpeg", "pid": 98359, "rss_mb": 47.098, "state": "S (sleeping)", "threads": 1}, "restart_count": 546, "restart_count_per_hour": 24, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 547}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.597, "sample_count": 566, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:49:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 34.618, "last_frame_pushed_at": "2026-07-01T10:49:33+09:00", "last_frame_read_age_sec": 34.769, "last_frame_read_at": "2026-07-01T10:49:33+09:00", "mjpeg": {"age_sec": 34.602, "exists": true, "mtime": "2026-07-01T10:49:33.324977+09:00", "mtime_epoch": 1782870573.324977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169815}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:50:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10992.801, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 498, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:49:33+09:00", "last_error_summary": "[h264 @ 0x55a1a842db40] error while decoding MB 66 25, bytestream -44", "last_exit_at": "2026-07-01T10:49:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 32.074, "last_started_at": "2026-07-01T10:49:35+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 546, "restart_count_per_hour": 24, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 547}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.597, "sample_count": 566, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:49:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 34.627, "last_frame_pushed_at": "2026-07-01T10:49:33+09:00", "last_frame_read_age_sec": 34.777, "last_frame_read_at": "2026-07-01T10:49:33+09:00", "mjpeg": {"age_sec": 34.611, "exists": true, "mtime": "2026-07-01T10:49:33.324977+09:00", "mtime_epoch": 1782870573.324977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169815}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:50:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11001.754, "limit_mb": 8796093022207.996}, "detail": {"pid": 98361, "restart_count": 547, "start_count": 548}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 498, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:49:33+09:00", "last_error_summary": "[h264 @ 0x55a1a842db40] error while decoding MB 66 25, bytestream -44", "last_exit_at": "2026-07-01T10:49:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:50:12+09:00", "pid": 98361, "process": {"exists": true, "fd_count": 4, "hwm_mb": 46.797, "name": "ffmpeg", "pid": 98361, "rss_mb": 46.797, "state": "S (sleeping)", "threads": 1}, "restart_count": 547, "restart_count_per_hour": 25, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 548}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.395, "sample_count": 522, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:49:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 41.162, "last_frame_pushed_at": "2026-07-01T10:49:33+09:00", "last_frame_read_age_sec": 41.313, "last_frame_read_at": "2026-07-01T10:49:33+09:00", "mjpeg": {"age_sec": 41.146, "exists": true, "mtime": "2026-07-01T10:49:33.324977+09:00", "mtime_epoch": 1782870573.324977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169815}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:50:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10992.449, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:51:28+09:00", "last_error_summary": "[in#0/rtsp @ 0x55de73eece00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55de73eece00] Error retrieving a packet from demuxer: Input/output error", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 499, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:51:28+09:00", "last_error_summary": "[in#0/rtsp @ 0x55de73eece00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55de73eece00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:51:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 76.208, "last_started_at": "2026-07-01T10:50:12+09:00", "pid": 98361, "process": {"exists": false, "pid": 98361}, "restart_count": 547, "restart_count_per_hour": 25, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 548}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.132, "sample_count": 360, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:51:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T10:51:28+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-01T10:51:28+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-01T10:51:28.559049+09:00", "mtime_epoch": 1782870688.559049, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167355}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:51:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.428, "pushed": 0.428, "read": 0.428}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.92, "count": 12, "max_ms": 38.474, "p95_ms": 38.474}, "gstreamer_push": {"avg_ms": 1.083, "count": 12, "max_ms": 1.131, "p95_ms": 1.131}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.198, "count": 12, "max_ms": 33.62, "p95_ms": 33.62}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.25, "count": 12, "max_ms": 62.62, "p95_ms": 62.62}, "yolo11_640_fire_11class": {"avg_ms": 43.211, "count": 12, "max_ms": 45.482, "p95_ms": 45.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.309, "count": 12, "max_ms": 55.414, "p95_ms": 55.414}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.17, "count": 12, "max_ms": 162.003, "p95_ms": 162.003}, "read_interval": {"avg_ms": 2734.521, "count": 12, "max_ms": 17584.715, "p95_ms": 17584.715}, "triton_wait": {"avg_ms": 55.263, "count": 12, "max_ms": 62.633, "p95_ms": 62.633}, "visualize": {"avg_ms": 4.457, "count": 12, "max_ms": 5.31, "p95_ms": 5.31}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.019, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10992.566, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 499, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:51:28+09:00", "last_error_summary": "[in#0/rtsp @ 0x55de73eece00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55de73eece00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:51:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 76.211, "last_started_at": "2026-07-01T10:50:12+09:00", "pid": 98361, "process": {"exists": false, "pid": 98361}, "restart_count": 547, "restart_count_per_hour": 25, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 548}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.132, "sample_count": 360, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:51:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T10:51:28+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-01T10:51:28+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-01T10:51:28.559049+09:00", "mtime_epoch": 1782870688.559049, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167355}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:51:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.428, "pushed": 0.428, "read": 0.428}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.92, "count": 12, "max_ms": 38.474, "p95_ms": 38.474}, "gstreamer_push": {"avg_ms": 1.083, "count": 12, "max_ms": 1.131, "p95_ms": 1.131}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.198, "count": 12, "max_ms": 33.62, "p95_ms": 33.62}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.25, "count": 12, "max_ms": 62.62, "p95_ms": 62.62}, "yolo11_640_fire_11class": {"avg_ms": 43.211, "count": 12, "max_ms": 45.482, "p95_ms": 45.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.309, "count": 12, "max_ms": 55.414, "p95_ms": 55.414}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.17, "count": 12, "max_ms": 162.003, "p95_ms": 162.003}, "read_interval": {"avg_ms": 2734.521, "count": 12, "max_ms": 17584.715, "p95_ms": 17584.715}, "triton_wait": {"avg_ms": 55.263, "count": 12, "max_ms": 62.633, "p95_ms": 62.633}, "visualize": {"avg_ms": 4.457, "count": 12, "max_ms": 5.31, "p95_ms": 5.31}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.022, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11002.789, "limit_mb": 8796093022207.996}, "detail": {"pid": 98468, "restart_count": 548, "start_count": 549}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 499, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:51:28+09:00", "last_error_summary": "[in#0/rtsp @ 0x55de73eece00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55de73eece00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:51:30+09:00", "pid": 98468, "process": {"exists": true, "fd_count": 4, "hwm_mb": 46.914, "name": "ffmpeg", "pid": 98468, "rss_mb": 46.914, "state": "S (sleeping)", "threads": 1}, "restart_count": 548, "restart_count_per_hour": 26, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 549}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.913, "sample_count": 332, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:51:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.226, "last_frame_pushed_at": "2026-07-01T10:51:28+09:00", "last_frame_read_age_sec": 4.35, "last_frame_read_at": "2026-07-01T10:51:28+09:00", "mjpeg": {"age_sec": 4.209, "exists": true, "mtime": "2026-07-01T10:51:28.691051+09:00", "mtime_epoch": 1782870688.6910512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167876}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:51:32+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 10, "pushed_frames": 10, "read_frames": 10, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.435, "pushed": 0.435, "read": 0.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 30.191, "count": 10, "max_ms": 38.474, "p95_ms": 38.474}, "gstreamer_push": {"avg_ms": 1.091, "count": 10, "max_ms": 1.131, "p95_ms": 1.131}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.224, "count": 10, "max_ms": 33.62, "p95_ms": 33.62}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.321, "count": 10, "max_ms": 62.62, "p95_ms": 62.62}, "yolo11_640_fire_11class": {"avg_ms": 43.05, "count": 10, "max_ms": 45.482, "p95_ms": 45.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.932, "count": 10, "max_ms": 55.414, "p95_ms": 55.414}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.715, "count": 10, "max_ms": 136.045, "p95_ms": 136.045}, "read_interval": {"avg_ms": 2771.801, "count": 10, "max_ms": 17584.715, "p95_ms": 17584.715}, "triton_wait": {"avg_ms": 54.333, "count": 10, "max_ms": 62.633, "p95_ms": 62.633}, "visualize": {"avg_ms": 4.346, "count": 10, "max_ms": 4.524, "p95_ms": 4.524}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.996, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11020.48, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 499, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:51:28+09:00", "last_error_summary": "[in#0/rtsp @ 0x55de73eece00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55de73eece00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 32.059, "last_started_at": "2026-07-01T10:51:30+09:00", "pid": 98468, "process": {"exists": true, "fd_count": 5, "hwm_mb": 64.852, "name": "ffmpeg", "pid": 98468, "rss_mb": 64.852, "state": "S (sleeping)", "threads": 19}, "restart_count": 548, "restart_count_per_hour": 26, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 549}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.68, "sample_count": 240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:51:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 34.284, "last_frame_pushed_at": "2026-07-01T10:51:28+09:00", "last_frame_read_age_sec": 34.408, "last_frame_read_at": "2026-07-01T10:51:28+09:00", "mjpeg": {"age_sec": 34.267, "exists": true, "mtime": "2026-07-01T10:51:28.691051+09:00", "mtime_epoch": 1782870688.6910512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167876}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:52:02+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11017.965, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 499, "h264_decode_error_count": 1444, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:51:28+09:00", "last_error_summary": "[in#0/rtsp @ 0x55de73eece00] Error during demuxing: Input/output error [in#0/rtsp @ 0x55de73eece00] Error retrieving a packet from demuxer: Input/output error", "last_exit_at": "2026-07-01T10:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 32.066, "last_started_at": "2026-07-01T10:51:30+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 548, "restart_count_per_hour": 26, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 549}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.68, "sample_count": 240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:51:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 34.292, "last_frame_pushed_at": "2026-07-01T10:51:28+09:00", "last_frame_read_age_sec": 34.416, "last_frame_read_at": "2026-07-01T10:51:28+09:00", "mjpeg": {"age_sec": 34.274, "exists": true, "mtime": "2026-07-01T10:51:28.691051+09:00", "mtime_epoch": 1782870688.6910512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167876}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:52:02+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10991.797, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562073ab2300] error while decoding MB 50 19, bytestream -32 [h264 @ 0x562073ab5280] error while decoding MB 50 19, bytestream -32", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 500, "h264_decode_error_count": 1446, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:52:02+09:00", "last_error_summary": "[h264 @ 0x562073ab2300] error while decoding MB 50 19, bytestream -32 [h264 @ 0x562073ab5280] error while decoding MB 50 19, bytestream -32", "last_exit_at": "2026-07-01T10:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 32.075, "last_started_at": "2026-07-01T10:51:30+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 548, "restart_count_per_hour": 26, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 549}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.68, "sample_count": 240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:51:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 34.3, "last_frame_pushed_at": "2026-07-01T10:51:28+09:00", "last_frame_read_age_sec": 34.424, "last_frame_read_at": "2026-07-01T10:51:28+09:00", "mjpeg": {"age_sec": 34.283, "exists": true, "mtime": "2026-07-01T10:51:28.691051+09:00", "mtime_epoch": 1782870688.6910512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167876}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:52:02+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11002.996, "limit_mb": 8796093022207.996}, "detail": {"pid": 98488, "restart_count": 549, "start_count": 550}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 500, "h264_decode_error_count": 1446, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:52:02+09:00", "last_error_summary": "[h264 @ 0x562073ab2300] error while decoding MB 50 19, bytestream -32 [h264 @ 0x562073ab5280] error while decoding MB 50 19, bytestream -32", "last_exit_at": "2026-07-01T10:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:52:07+09:00", "pid": 98488, "process": {"exists": true, "fd_count": 4, "hwm_mb": 46.906, "name": "ffmpeg", "pid": 98488, "rss_mb": 46.906, "state": "S (sleeping)", "threads": 1}, "restart_count": 549, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 550}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.796, "sample_count": 230, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:51:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 40.387, "last_frame_pushed_at": "2026-07-01T10:51:28+09:00", "last_frame_read_age_sec": 40.511, "last_frame_read_at": "2026-07-01T10:51:28+09:00", "mjpeg": {"age_sec": 40.37, "exists": true, "mtime": "2026-07-01T10:51:28.691051+09:00", "mtime_epoch": 1782870688.6910512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167876}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:52:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11103.219, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 500, "h264_decode_error_count": 1446, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:52:02+09:00", "last_error_summary": "[h264 @ 0x562073ab2300] error while decoding MB 50 19, bytestream -32 [h264 @ 0x562073ab5280] error while decoding MB 50 19, bytestream -32", "last_exit_at": "2026-07-01T10:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 41.903, "last_started_at": "2026-07-01T10:52:07+09:00", "pid": 98488, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.578, "name": "ffmpeg", "pid": 98488, "rss_mb": 142.578, "state": "S (sleeping)", "threads": 106}, "restart_count": 549, "restart_count_per_hour": 26, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 550}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.388, "sample_count": 192, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449350, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:52:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 30.054, "last_frame_pushed_at": "2026-07-01T10:52:18+09:00", "last_frame_read_age_sec": 30.192, "last_frame_read_at": "2026-07-01T10:52:18+09:00", "mjpeg": {"age_sec": 30.039, "exists": true, "mtime": "2026-07-01T10:52:18.923952+09:00", "mtime_epoch": 1782870738.9239516, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168522}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:52:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11102.836, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 500, "h264_decode_error_count": 1446, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:52:02+09:00", "last_error_summary": "[h264 @ 0x562073ab2300] error while decoding MB 50 19, bytestream -32 [h264 @ 0x562073ab5280] error while decoding MB 50 19, bytestream -32", "last_exit_at": "2026-07-01T10:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 41.912, "last_started_at": "2026-07-01T10:52:07+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 549, "restart_count_per_hour": 26, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 550}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.388, "sample_count": 192, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449350, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:52:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 30.063, "last_frame_pushed_at": "2026-07-01T10:52:18+09:00", "last_frame_read_age_sec": 30.201, "last_frame_read_at": "2026-07-01T10:52:18+09:00", "mjpeg": {"age_sec": 30.048, "exists": true, "mtime": "2026-07-01T10:52:18.923952+09:00", "mtime_epoch": 1782870738.9239516, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168522}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:52:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10993.637, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560cba8d8740] error while decoding MB 74 44, bytestream -22", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 501, "h264_decode_error_count": 1447, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:52:48+09:00", "last_error_summary": "[h264 @ 0x560cba8d8740] error while decoding MB 74 44, bytestream -22", "last_exit_at": "2026-07-01T10:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 41.934, "last_started_at": "2026-07-01T10:52:07+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 549, "restart_count_per_hour": 26, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 550}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.388, "sample_count": 192, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449350, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:52:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 30.086, "last_frame_pushed_at": "2026-07-01T10:52:18+09:00", "last_frame_read_age_sec": 30.224, "last_frame_read_at": "2026-07-01T10:52:18+09:00", "mjpeg": {"age_sec": 30.07, "exists": true, "mtime": "2026-07-01T10:52:18.923952+09:00", "mtime_epoch": 1782870738.9239516, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168522}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:52:48+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11002.281, "limit_mb": 8796093022207.996}, "detail": {"pid": 98595, "restart_count": 550, "start_count": 551}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 501, "h264_decode_error_count": 1447, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:52:48+09:00", "last_error_summary": "[h264 @ 0x560cba8d8740] error while decoding MB 74 44, bytestream -22", "last_exit_at": "2026-07-01T10:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:52:51+09:00", "pid": 98595, "process": {"exists": true, "fd_count": 4, "hwm_mb": 44.09, "name": "ffmpeg", "pid": 98595, "rss_mb": 44.09, "state": "S (sleeping)", "threads": 1}, "restart_count": 550, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 551}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 175.377, "sample_count": 186, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449350, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:52:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 34.521, "last_frame_pushed_at": "2026-07-01T10:52:18+09:00", "last_frame_read_age_sec": 34.659, "last_frame_read_at": "2026-07-01T10:52:18+09:00", "mjpeg": {"age_sec": 34.506, "exists": true, "mtime": "2026-07-01T10:52:18.923952+09:00", "mtime_epoch": 1782870738.9239516, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168522}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:52:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10993.793, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55dcd49e1640] error while decoding MB 78 32, bytestream -17 [in#0/rtsp @ 0x55dcd3e7ce00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x55dcd3e7ce00] Error retrieving a packet from demuxer: Connection reset by peer", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 502, "h264_decode_error_count": 1448, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:54:14+09:00", "last_error_summary": "[h264 @ 0x55dcd49e1640] error while decoding MB 78 32, bytestream -17 [in#0/rtsp @ 0x55dcd3e7ce00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x55dcd3e7ce00] Error retrieving a packet from demuxer: Connection reset by peer", "last_exit_at": "2026-07-01T10:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 83.323, "last_started_at": "2026-07-01T10:52:51+09:00", "pid": 98595, "process": {"exists": false, "pid": 98595}, "restart_count": 550, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 551}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.989, "sample_count": 219, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449443, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:54:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-01T10:54:14+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T10:54:14+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T10:54:14.714022+09:00", "mtime_epoch": 1782870854.7140217, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172285}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:54:14+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.895, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.469, "pushed": 0.469, "read": 0.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.295, "count": 14, "max_ms": 35.143, "p95_ms": 35.143}, "gstreamer_push": {"avg_ms": 1.004, "count": 14, "max_ms": 1.037, "p95_ms": 1.037}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.752, "count": 14, "max_ms": 38.669, "p95_ms": 38.669}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.783, "count": 14, "max_ms": 72.463, "p95_ms": 72.463}, "yolo11_640_fire_11class": {"avg_ms": 46.581, "count": 14, "max_ms": 60.055, "p95_ms": 60.055}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.056, "count": 14, "max_ms": 73.736, "p95_ms": 73.736}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.119, "count": 14, "max_ms": 165.401, "p95_ms": 165.401}, "read_interval": {"avg_ms": 2140.923, "count": 14, "max_ms": 17971.452, "p95_ms": 17971.452}, "triton_wait": {"avg_ms": 60.027, "count": 14, "max_ms": 73.748, "p95_ms": 73.748}, "visualize": {"avg_ms": 5.292, "count": 14, "max_ms": 5.785, "p95_ms": 5.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.863, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10992.695, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:54:14+09:00", "last_error_summary": "[h264 @ 0x55dcd49e1640] error while decoding MB 78 32, bytestream -17 [in#0/rtsp @ 0x55dcd3e7ce00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x55dcd3e7ce00] Error retrieving a packet from demuxer: Connection reset by peer", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 502, "h264_decode_error_count": 1448, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:54:14+09:00", "last_error_summary": "[h264 @ 0x55dcd49e1640] error while decoding MB 78 32, bytestream -17 [in#0/rtsp @ 0x55dcd3e7ce00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x55dcd3e7ce00] Error retrieving a packet from demuxer: Connection reset by peer", "last_exit_at": "2026-07-01T10:54:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 83.433, "last_started_at": "2026-07-01T10:52:51+09:00", "pid": 98595, "process": {"exists": false, "pid": 98595}, "restart_count": 550, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 551}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.989, "sample_count": 220, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449444, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:54:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T10:54:14+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-01T10:54:14+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-01T10:54:14.714022+09:00", "mtime_epoch": 1782870854.7140217, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172285}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:54:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.895, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 15, "pushed_frames": 15, "read_frames": 14, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.5, "pushed": 0.5, "read": 0.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.479, "count": 14, "max_ms": 35.143, "p95_ms": 35.143}, "gstreamer_push": {"avg_ms": 1.003, "count": 15, "max_ms": 1.037, "p95_ms": 1.037}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.893, "count": 14, "max_ms": 38.669, "p95_ms": 38.669}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.561, "count": 14, "max_ms": 72.463, "p95_ms": 72.463}, "yolo11_640_fire_11class": {"avg_ms": 46.689, "count": 14, "max_ms": 60.055, "p95_ms": 60.055}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.121, "count": 14, "max_ms": 73.736, "p95_ms": 73.736}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.095, "count": 15, "max_ms": 165.401, "p95_ms": 165.401}, "read_interval": {"avg_ms": 2140.923, "count": 14, "max_ms": 17971.452, "p95_ms": 17971.452}, "triton_wait": {"avg_ms": 59.805, "count": 14, "max_ms": 73.748, "p95_ms": 73.748}, "visualize": {"avg_ms": 5.263, "count": 14, "max_ms": 5.785, "p95_ms": 5.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10992.832, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 502, "h264_decode_error_count": 1448, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:54:14+09:00", "last_error_summary": "[h264 @ 0x55dcd49e1640] error while decoding MB 78 32, bytestream -17 [in#0/rtsp @ 0x55dcd3e7ce00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x55dcd3e7ce00] Error retrieving a packet from demuxer: Connection reset by peer", "last_exit_at": "2026-07-01T10:54:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 83.436, "last_started_at": "2026-07-01T10:52:51+09:00", "pid": 98595, "process": {"exists": false, "pid": 98595}, "restart_count": 550, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 551}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.989, "sample_count": 220, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449444, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:54:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T10:54:14+09:00", "last_frame_read_age_sec": 0.162, "last_frame_read_at": "2026-07-01T10:54:14+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-01T10:54:14.714022+09:00", "mtime_epoch": 1782870854.7140217, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172285}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:54:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.895, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 15, "pushed_frames": 15, "read_frames": 14, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.5, "pushed": 0.5, "read": 0.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.479, "count": 14, "max_ms": 35.143, "p95_ms": 35.143}, "gstreamer_push": {"avg_ms": 1.003, "count": 15, "max_ms": 1.037, "p95_ms": 1.037}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.893, "count": 14, "max_ms": 38.669, "p95_ms": 38.669}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.561, "count": 14, "max_ms": 72.463, "p95_ms": 72.463}, "yolo11_640_fire_11class": {"avg_ms": 46.689, "count": 14, "max_ms": 60.055, "p95_ms": 60.055}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.121, "count": 14, "max_ms": 73.736, "p95_ms": 73.736}}, "truncated_count": 0}, "processing_total": {"avg_ms": 154.095, "count": 15, "max_ms": 165.401, "p95_ms": 165.401}, "read_interval": {"avg_ms": 2140.923, "count": 14, "max_ms": 17971.452, "p95_ms": 17971.452}, "triton_wait": {"avg_ms": 59.805, "count": 14, "max_ms": 73.748, "p95_ms": 73.748}, "visualize": {"avg_ms": 5.263, "count": 14, "max_ms": 5.785, "p95_ms": 5.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11096.309, "limit_mb": 8796093022207.996}, "detail": {"pid": 98702, "restart_count": 551, "start_count": 552}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 502, "h264_decode_error_count": 1448, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:54:14+09:00", "last_error_summary": "[h264 @ 0x55dcd49e1640] error while decoding MB 78 32, bytestream -17 [in#0/rtsp @ 0x55dcd3e7ce00] Error during demuxing: Connection reset by peer [in#0/rtsp @ 0x55dcd3e7ce00] Error retrieving a packet from demuxer: Connection reset by peer", "last_exit_at": "2026-07-01T10:54:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:54:17+09:00", "pid": 98702, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.398, "name": "ffmpeg", "pid": 98702, "rss_mb": 133.398, "state": "S (sleeping)", "threads": 106}, "restart_count": 551, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 552}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.989, "sample_count": 219, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449444, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:54:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.239, "last_frame_pushed_at": "2026-07-01T10:54:14+09:00", "last_frame_read_age_sec": 4.395, "last_frame_read_at": "2026-07-01T10:54:14+09:00", "mjpeg": {"age_sec": 4.223, "exists": true, "mtime": "2026-07-01T10:54:14.874024+09:00", "mtime_epoch": 1782870854.8740244, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172223}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:54:19+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.039, "name": "celery", "pid": 267, "rss_mb": 5056.895, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.576, "pushed": 0.576, "read": 0.576}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.479, "count": 14, "max_ms": 35.143, "p95_ms": 35.143}, "gstreamer_push": {"avg_ms": 1.002, "count": 14, "max_ms": 1.037, "p95_ms": 1.037}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.893, "count": 14, "max_ms": 38.669, "p95_ms": 38.669}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.561, "count": 14, "max_ms": 72.463, "p95_ms": 72.463}, "yolo11_640_fire_11class": {"avg_ms": 46.689, "count": 14, "max_ms": 60.055, "p95_ms": 60.055}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.121, "count": 14, "max_ms": 73.736, "p95_ms": 73.736}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.768, "count": 14, "max_ms": 165.401, "p95_ms": 165.401}, "read_interval": {"avg_ms": 2140.923, "count": 14, "max_ms": 17971.452, "p95_ms": 17971.452}, "triton_wait": {"avg_ms": 59.805, "count": 14, "max_ms": 73.748, "p95_ms": 73.748}, "visualize": {"avg_ms": 5.263, "count": 14, "max_ms": 5.785, "p95_ms": 5.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 24.319, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10995.172, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f4f2946f00] error while decoding MB 3 23, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 503, "h264_decode_error_count": 1449, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:55:57+09:00", "last_error_summary": "[h264 @ 0x55f4f2946f00] error while decoding MB 3 23, bytestream -5", "last_exit_at": "2026-07-01T10:54:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 100.335, "last_started_at": "2026-07-01T10:54:17+09:00", "pid": 98702, "process": {"exists": false, "pid": 98702}, "restart_count": 551, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 552}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.768, "sample_count": 599, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449926, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:55:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T10:55:57+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T10:55:57+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T10:55:57.391852+09:00", "mtime_epoch": 1782870957.3918517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171400}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:55:57+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.121, "name": "celery", "pid": 267, "rss_mb": 5057.121, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 191, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.384, "pushed": 6.384, "read": 6.384}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.521, "count": 191, "max_ms": 41.322, "p95_ms": 35.81}, "gstreamer_push": {"avg_ms": 1.082, "count": 191, "max_ms": 1.46, "p95_ms": 1.152}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.719, "count": 191, "max_ms": 42.616, "p95_ms": 35.984}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.367, "count": 191, "max_ms": 92.091, "p95_ms": 86.578}, "yolo11_640_fire_11class": {"avg_ms": 50.189, "count": 191, "max_ms": 69.304, "p95_ms": 63.906}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.041, "count": 191, "max_ms": 87.416, "p95_ms": 79.38}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.24, "count": 191, "max_ms": 187.881, "p95_ms": 170.078}, "read_interval": {"avg_ms": 157.051, "count": 191, "max_ms": 197.666, "p95_ms": 180.708}, "triton_wait": {"avg_ms": 68.91, "count": 191, "max_ms": 92.103, "p95_ms": 86.591}, "visualize": {"avg_ms": 4.619, "count": 191, "max_ms": 6.55, "p95_ms": 5.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10994.562, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:55:57+09:00", "last_error_summary": "[h264 @ 0x55f4f2946f00] error while decoding MB 3 23, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 503, "h264_decode_error_count": 1449, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:55:57+09:00", "last_error_summary": "[h264 @ 0x55f4f2946f00] error while decoding MB 3 23, bytestream -5", "last_exit_at": "2026-07-01T10:55:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 100.447, "last_started_at": "2026-07-01T10:54:17+09:00", "pid": 98702, "process": {"exists": false, "pid": 98702}, "restart_count": 551, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 552}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.768, "sample_count": 600, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449927, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:55:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T10:55:57+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-01T10:55:57+09:00", "mjpeg": {"age_sec": 0.152, "exists": true, "mtime": "2026-07-01T10:55:57.391852+09:00", "mtime_epoch": 1782870957.3918517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171400}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:55:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.121, "name": "celery", "pid": 267, "rss_mb": 5057.121, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.394, "pushed": 6.394, "read": 6.36}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.549, "count": 191, "max_ms": 41.322, "p95_ms": 35.863}, "gstreamer_push": {"avg_ms": 1.081, "count": 191, "max_ms": 1.46, "p95_ms": 1.152}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.736, "count": 191, "max_ms": 42.616, "p95_ms": 35.984}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.365, "count": 191, "max_ms": 92.091, "p95_ms": 86.578}, "yolo11_640_fire_11class": {"avg_ms": 50.159, "count": 191, "max_ms": 69.304, "p95_ms": 63.906}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.044, "count": 191, "max_ms": 87.416, "p95_ms": 79.38}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.382, "count": 191, "max_ms": 187.881, "p95_ms": 170.078}, "read_interval": {"avg_ms": 157.135, "count": 190, "max_ms": 197.666, "p95_ms": 180.708}, "triton_wait": {"avg_ms": 68.908, "count": 191, "max_ms": 92.103, "p95_ms": 86.591}, "visualize": {"avg_ms": 4.624, "count": 191, "max_ms": 6.55, "p95_ms": 5.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10994.555, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 503, "h264_decode_error_count": 1449, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:55:57+09:00", "last_error_summary": "[h264 @ 0x55f4f2946f00] error while decoding MB 3 23, bytestream -5", "last_exit_at": "2026-07-01T10:55:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 100.451, "last_started_at": "2026-07-01T10:54:17+09:00", "pid": 98702, "process": {"exists": false, "pid": 98702}, "restart_count": 551, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 552}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.768, "sample_count": 600, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449927, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:55:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T10:55:57+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T10:55:57+09:00", "mjpeg": {"age_sec": 0.155, "exists": true, "mtime": "2026-07-01T10:55:57.391852+09:00", "mtime_epoch": 1782870957.3918517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171400}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:55:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.121, "name": "celery", "pid": 267, "rss_mb": 5057.121, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.393, "pushed": 6.393, "read": 6.359}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.549, "count": 191, "max_ms": 41.322, "p95_ms": 35.863}, "gstreamer_push": {"avg_ms": 1.081, "count": 191, "max_ms": 1.46, "p95_ms": 1.152}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.736, "count": 191, "max_ms": 42.616, "p95_ms": 35.984}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.365, "count": 191, "max_ms": 92.091, "p95_ms": 86.578}, "yolo11_640_fire_11class": {"avg_ms": 50.159, "count": 191, "max_ms": 69.304, "p95_ms": 63.906}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.044, "count": 191, "max_ms": 87.416, "p95_ms": 79.38}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.382, "count": 191, "max_ms": 187.881, "p95_ms": 170.078}, "read_interval": {"avg_ms": 157.135, "count": 190, "max_ms": 197.666, "p95_ms": 180.708}, "triton_wait": {"avg_ms": 68.908, "count": 191, "max_ms": 92.103, "p95_ms": 86.591}, "visualize": {"avg_ms": 4.624, "count": 191, "max_ms": 6.55, "p95_ms": 5.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11099.539, "limit_mb": 8796093022207.996}, "detail": {"pid": 98809, "restart_count": 552, "start_count": 553}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 503, "h264_decode_error_count": 1449, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:55:57+09:00", "last_error_summary": "[h264 @ 0x55f4f2946f00] error while decoding MB 3 23, bytestream -5", "last_exit_at": "2026-07-01T10:56:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:56:00+09:00", "pid": 98809, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.207, "name": "ffmpeg", "pid": 98809, "rss_mb": 136.207, "state": "S (sleeping)", "threads": 106}, "restart_count": 552, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 553}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.768, "sample_count": 598, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1449927, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:55:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.68, "last_frame_pushed_at": "2026-07-01T10:55:57+09:00", "last_frame_read_age_sec": 4.84, "last_frame_read_at": "2026-07-01T10:55:57+09:00", "mjpeg": {"age_sec": 4.661, "exists": true, "mtime": "2026-07-01T10:55:57.559855+09:00", "mtime_epoch": 1782870957.5598547, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170442}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:56:02+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.121, "name": "celery", "pid": 267, "rss_mb": 5057.121, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 164, "pushed_frames": 164, "read_frames": 163, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.471, "pushed": 5.471, "read": 5.438}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.479, "count": 163, "max_ms": 41.322, "p95_ms": 35.776}, "gstreamer_push": {"avg_ms": 1.093, "count": 164, "max_ms": 1.46, "p95_ms": 1.152}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.815, "count": 163, "max_ms": 42.616, "p95_ms": 35.984}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.578, "count": 163, "max_ms": 89.408, "p95_ms": 86.578}, "yolo11_640_fire_11class": {"avg_ms": 49.959, "count": 163, "max_ms": 69.304, "p95_ms": 63.285}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.934, "count": 163, "max_ms": 85.388, "p95_ms": 79.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 144.286, "count": 164, "max_ms": 180.06, "p95_ms": 163.008}, "read_interval": {"avg_ms": 155.092, "count": 163, "max_ms": 190.131, "p95_ms": 174.283}, "triton_wait": {"avg_ms": 69.12, "count": 163, "max_ms": 89.42, "p95_ms": 86.59}, "visualize": {"avg_ms": 4.558, "count": 163, "max_ms": 6.55, "p95_ms": 5.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10997.957, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564d1f7e6cc0] cabac decode of qscale diff failed at 50 30 [h264 @ 0x564d1f7e6cc0] error while decoding MB 50 30, bytestream 53936 [h264 @ 0x564d1f7ee240] error while decoding MB 24 35, bytestream -5 [h264 @ 0x564d1f7ee240] error while decoding MB 35 40, bytestream -9 [h264 @ 0x564d20298bc0] reference picture missing during reorder [h264 @ 0x564d20298bc0] Missing reference picture, default is 65537 [h264 @ 0x564d20367a40] mmco: unref short failure [h264 @ 0x564d20367a40] illegal short t", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 504, "h264_decode_error_count": 1456, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:58:57+09:00", "last_error_summary": "[h264 @ 0x564d1f7e6cc0] cabac decode of qscale diff failed at 50 30 [h264 @ 0x564d1f7e6cc0] error while decoding MB 50 30, bytestream 53936 [h264 @ 0x564d1f7ee240] error while decoding MB 24 35, bytestream -5 [h264 @ 0x564d1f7ee240] error while decoding MB 35 40, bytestream -9 [h264 @ 0x564d20298bc0] reference picture missing during reorder [h264 @ 0x564d20298bc0] Missing reference picture, default is 65537 [h264 @ 0x564d20367a40] mmco: unref short failure [h264 @ 0x564d20367a40] illegal short t", "last_exit_at": "2026-07-01T10:56:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 177.728, "last_started_at": "2026-07-01T10:56:00+09:00", "pid": 98809, "process": {"exists": false, "pid": 98809}, "restart_count": 552, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 553}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.259, "sample_count": 1582, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451016, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:58:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T10:58:57+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T10:58:57+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T10:58:57.901359+09:00", "mtime_epoch": 1782871137.9013593, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173720}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:58:57+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.652, "name": "celery", "pid": 267, "rss_mb": 5057.652, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 183, "pushed_frames": 183, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.134, "pushed": 6.134, "read": 6.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.351, "count": 183, "max_ms": 44.092, "p95_ms": 38.143}, "gstreamer_push": {"avg_ms": 1.041, "count": 183, "max_ms": 1.481, "p95_ms": 1.274}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.634, "count": 183, "max_ms": 41.266, "p95_ms": 35.448}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.032, "count": 183, "max_ms": 90.377, "p95_ms": 86.529}, "yolo11_640_fire_11class": {"avg_ms": 48.958, "count": 183, "max_ms": 66.019, "p95_ms": 63.626}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.033, "count": 183, "max_ms": 84.597, "p95_ms": 81.714}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.949, "count": 183, "max_ms": 188.294, "p95_ms": 181.691}, "read_interval": {"avg_ms": 163.741, "count": 183, "max_ms": 369.716, "p95_ms": 191.327}, "triton_wait": {"avg_ms": 65.246, "count": 183, "max_ms": 90.39, "p95_ms": 86.541}, "visualize": {"avg_ms": 4.853, "count": 183, "max_ms": 6.714, "p95_ms": 6.277}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.833, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10997.234, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T10:58:57+09:00", "last_error_summary": "[h264 @ 0x564d1f7e6cc0] cabac decode of qscale diff failed at 50 30 [h264 @ 0x564d1f7e6cc0] error while decoding MB 50 30, bytestream 53936 [h264 @ 0x564d1f7ee240] error while decoding MB 24 35, bytestream -5 [h264 @ 0x564d1f7ee240] error while decoding MB 35 40, bytestream -9 [h264 @ 0x564d20298bc0] reference picture missing during reorder [h264 @ 0x564d20298bc0] Missing reference picture, default is 65537 [h264 @ 0x564d20367a40] mmco: unref short failure [h264 @ 0x564d20367a40] illegal short t", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 504, "h264_decode_error_count": 1456, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:58:57+09:00", "last_error_summary": "[h264 @ 0x564d1f7e6cc0] cabac decode of qscale diff failed at 50 30 [h264 @ 0x564d1f7e6cc0] error while decoding MB 50 30, bytestream 53936 [h264 @ 0x564d1f7ee240] error while decoding MB 24 35, bytestream -5 [h264 @ 0x564d1f7ee240] error while decoding MB 35 40, bytestream -9 [h264 @ 0x564d20298bc0] reference picture missing during reorder [h264 @ 0x564d20298bc0] Missing reference picture, default is 65537 [h264 @ 0x564d20367a40] mmco: unref short failure [h264 @ 0x564d20367a40] illegal short t", "last_exit_at": "2026-07-01T10:58:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 177.851, "last_started_at": "2026-07-01T10:56:00+09:00", "pid": 98809, "process": {"exists": false, "pid": 98809}, "restart_count": 552, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 553}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.259, "sample_count": 1583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451017, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:58:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T10:58:58+09:00", "last_frame_read_age_sec": 0.176, "last_frame_read_at": "2026-07-01T10:58:57+09:00", "mjpeg": {"age_sec": 0.17, "exists": true, "mtime": "2026-07-01T10:58:57.901359+09:00", "mtime_epoch": 1782871137.9013593, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173720}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:58:58+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.652, "name": "celery", "pid": 267, "rss_mb": 5057.652, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.142, "pushed": 6.142, "read": 6.109}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.351, "count": 183, "max_ms": 44.092, "p95_ms": 38.143}, "gstreamer_push": {"avg_ms": 1.041, "count": 184, "max_ms": 1.481, "p95_ms": 1.274}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.638, "count": 183, "max_ms": 41.266, "p95_ms": 35.448}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.991, "count": 183, "max_ms": 90.377, "p95_ms": 86.529}, "yolo11_640_fire_11class": {"avg_ms": 48.951, "count": 183, "max_ms": 66.019, "p95_ms": 63.626}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.083, "count": 183, "max_ms": 84.597, "p95_ms": 81.714}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.052, "count": 184, "max_ms": 188.294, "p95_ms": 181.691}, "read_interval": {"avg_ms": 163.741, "count": 183, "max_ms": 369.716, "p95_ms": 191.327}, "triton_wait": {"avg_ms": 65.278, "count": 183, "max_ms": 90.39, "p95_ms": 86.541}, "visualize": {"avg_ms": 4.849, "count": 183, "max_ms": 6.714, "p95_ms": 6.277}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10997.227, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 504, "h264_decode_error_count": 1456, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:58:57+09:00", "last_error_summary": "[h264 @ 0x564d1f7e6cc0] cabac decode of qscale diff failed at 50 30 [h264 @ 0x564d1f7e6cc0] error while decoding MB 50 30, bytestream 53936 [h264 @ 0x564d1f7ee240] error while decoding MB 24 35, bytestream -5 [h264 @ 0x564d1f7ee240] error while decoding MB 35 40, bytestream -9 [h264 @ 0x564d20298bc0] reference picture missing during reorder [h264 @ 0x564d20298bc0] Missing reference picture, default is 65537 [h264 @ 0x564d20367a40] mmco: unref short failure [h264 @ 0x564d20367a40] illegal short t", "last_exit_at": "2026-07-01T10:58:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 177.855, "last_started_at": "2026-07-01T10:56:00+09:00", "pid": 98809, "process": {"exists": false, "pid": 98809}, "restart_count": 552, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 553}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.259, "sample_count": 1583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451017, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:58:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T10:58:58+09:00", "last_frame_read_age_sec": 0.18, "last_frame_read_at": "2026-07-01T10:58:57+09:00", "mjpeg": {"age_sec": 0.173, "exists": true, "mtime": "2026-07-01T10:58:57.901359+09:00", "mtime_epoch": 1782871137.9013593, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173720}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:58:58+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5057.652, "name": "celery", "pid": 267, "rss_mb": 5057.652, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.142, "pushed": 6.142, "read": 6.108}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.351, "count": 183, "max_ms": 44.092, "p95_ms": 38.143}, "gstreamer_push": {"avg_ms": 1.041, "count": 184, "max_ms": 1.481, "p95_ms": 1.274}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.638, "count": 183, "max_ms": 41.266, "p95_ms": 35.448}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.991, "count": 183, "max_ms": 90.377, "p95_ms": 86.529}, "yolo11_640_fire_11class": {"avg_ms": 48.951, "count": 183, "max_ms": 66.019, "p95_ms": 63.626}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.083, "count": 183, "max_ms": 84.597, "p95_ms": 81.714}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.052, "count": 184, "max_ms": 188.294, "p95_ms": 181.691}, "read_interval": {"avg_ms": 163.741, "count": 183, "max_ms": 369.716, "p95_ms": 191.327}, "triton_wait": {"avg_ms": 65.278, "count": 183, "max_ms": 90.39, "p95_ms": 86.541}, "visualize": {"avg_ms": 4.849, "count": 183, "max_ms": 6.714, "p95_ms": 6.277}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11102.645, "limit_mb": 8796093022207.996}, "detail": {"pid": 98916, "restart_count": 553, "start_count": 554}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 504, "h264_decode_error_count": 1456, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T10:58:57+09:00", "last_error_summary": "[h264 @ 0x564d1f7e6cc0] cabac decode of qscale diff failed at 50 30 [h264 @ 0x564d1f7e6cc0] error while decoding MB 50 30, bytestream 53936 [h264 @ 0x564d1f7ee240] error while decoding MB 24 35, bytestream -5 [h264 @ 0x564d1f7ee240] error while decoding MB 35 40, bytestream -9 [h264 @ 0x564d20298bc0] reference picture missing during reorder [h264 @ 0x564d20298bc0] Missing reference picture, default is 65537 [h264 @ 0x564d20367a40] mmco: unref short failure [h264 @ 0x564d20367a40] illegal short t", "last_exit_at": "2026-07-01T10:59:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T10:59:00+09:00", "pid": 98916, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.492, "name": "ffmpeg", "pid": 98916, "rss_mb": 136.492, "state": "S (sleeping)", "threads": 106}, "restart_count": 553, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 554}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.259, "sample_count": 1583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451017, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T10:58:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.149, "last_frame_pushed_at": "2026-07-01T10:58:58+09:00", "last_frame_read_age_sec": 4.322, "last_frame_read_at": "2026-07-01T10:58:57+09:00", "mjpeg": {"age_sec": 4.135, "exists": true, "mtime": "2026-07-01T10:58:58.081361+09:00", "mtime_epoch": 1782871138.081361, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169206}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T10:59:02+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5057.652, "name": "celery", "pid": 267, "rss_mb": 5057.652, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.377, "pushed": 5.377, "read": 5.343}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.233, "count": 160, "max_ms": 44.092, "p95_ms": 38.38}, "gstreamer_push": {"avg_ms": 1.063, "count": 161, "max_ms": 1.481, "p95_ms": 1.275}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.747, "count": 160, "max_ms": 41.266, "p95_ms": 35.571}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.044, "count": 160, "max_ms": 90.377, "p95_ms": 86.535}, "yolo11_640_fire_11class": {"avg_ms": 49.487, "count": 160, "max_ms": 66.019, "p95_ms": 63.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.78, "count": 160, "max_ms": 84.597, "p95_ms": 82.082}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.963, "count": 161, "max_ms": 185.235, "p95_ms": 177.972}, "read_interval": {"avg_ms": 161.228, "count": 160, "max_ms": 369.716, "p95_ms": 186.596}, "triton_wait": {"avg_ms": 66.33, "count": 160, "max_ms": 90.39, "p95_ms": 86.545}, "visualize": {"avg_ms": 4.675, "count": 160, "max_ms": 6.574, "p95_ms": 5.921}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12098}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10999.078, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558940aa8a80] error while decoding MB 24 25, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 505, "h264_decode_error_count": 1457, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:01:12+09:00", "last_error_summary": "[h264 @ 0x558940aa8a80] error while decoding MB 24 25, bytestream -12", "last_exit_at": "2026-07-01T10:59:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.4, "last_started_at": "2026-07-01T10:59:00+09:00", "pid": 98916, "process": {"exists": false, "pid": 98916}, "restart_count": 553, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 554}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.498, "sample_count": 1812, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451806, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:01:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T11:01:12+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T11:01:12+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T11:01:12.570690+09:00", "mtime_epoch": 1782871272.5706902, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171530}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:01:12+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5058.031, "name": "celery", "pid": 267, "rss_mb": 5058.031, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 171, "pushed_frames": 171, "read_frames": 171, "select_timeouts": 2, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.714, "pushed": 5.714, "read": 5.714}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.698, "count": 171, "max_ms": 41.254, "p95_ms": 35.721}, "gstreamer_push": {"avg_ms": 1.04, "count": 171, "max_ms": 1.308, "p95_ms": 1.144}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.428, "count": 171, "max_ms": 47.459, "p95_ms": 35.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.915, "count": 171, "max_ms": 89.809, "p95_ms": 87.29}, "yolo11_640_fire_11class": {"avg_ms": 48.283, "count": 171, "max_ms": 68.298, "p95_ms": 62.997}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.029, "count": 171, "max_ms": 85.88, "p95_ms": 82.468}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.541, "count": 171, "max_ms": 187.068, "p95_ms": 180.621}, "read_interval": {"avg_ms": 175.763, "count": 171, "max_ms": 2543.167, "p95_ms": 192.266}, "triton_wait": {"avg_ms": 66.143, "count": 171, "max_ms": 89.822, "p95_ms": 87.302}, "visualize": {"avg_ms": 4.809, "count": 171, "max_ms": 6.146, "p95_ms": 5.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12098}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10997.941, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:01:12+09:00", "last_error_summary": "[h264 @ 0x558940aa8a80] error while decoding MB 24 25, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 505, "h264_decode_error_count": 1457, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:01:12+09:00", "last_error_summary": "[h264 @ 0x558940aa8a80] error while decoding MB 24 25, bytestream -12", "last_exit_at": "2026-07-01T11:01:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.51, "last_started_at": "2026-07-01T10:59:00+09:00", "pid": 98916, "process": {"exists": false, "pid": 98916}, "restart_count": 553, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 554}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.498, "sample_count": 1812, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451807, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:01:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T11:01:12+09:00", "last_frame_read_age_sec": 0.161, "last_frame_read_at": "2026-07-01T11:01:12+09:00", "mjpeg": {"age_sec": 0.155, "exists": true, "mtime": "2026-07-01T11:01:12.570690+09:00", "mtime_epoch": 1782871272.5706902, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171530}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:01:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5058.031, "name": "celery", "pid": 267, "rss_mb": 5058.031, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 171, "pushed_frames": 171, "read_frames": 170, "select_timeouts": 2, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.725, "pushed": 5.725, "read": 5.691}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.692, "count": 171, "max_ms": 41.254, "p95_ms": 35.721}, "gstreamer_push": {"avg_ms": 1.04, "count": 171, "max_ms": 1.308, "p95_ms": 1.144}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.433, "count": 171, "max_ms": 47.459, "p95_ms": 35.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.766, "count": 171, "max_ms": 89.809, "p95_ms": 87.29}, "yolo11_640_fire_11class": {"avg_ms": 48.165, "count": 171, "max_ms": 68.298, "p95_ms": 62.997}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.859, "count": 171, "max_ms": 85.88, "p95_ms": 82.468}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.376, "count": 171, "max_ms": 187.068, "p95_ms": 180.153}, "read_interval": {"avg_ms": 175.66, "count": 170, "max_ms": 2543.167, "p95_ms": 192.136}, "triton_wait": {"avg_ms": 65.995, "count": 171, "max_ms": 89.822, "p95_ms": 87.302}, "visualize": {"avg_ms": 4.808, "count": 171, "max_ms": 6.146, "p95_ms": 5.509}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12098}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10997.934, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 505, "h264_decode_error_count": 1457, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:01:12+09:00", "last_error_summary": "[h264 @ 0x558940aa8a80] error while decoding MB 24 25, bytestream -12", "last_exit_at": "2026-07-01T11:01:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.513, "last_started_at": "2026-07-01T10:59:00+09:00", "pid": 98916, "process": {"exists": false, "pid": 98916}, "restart_count": 553, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 554}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.498, "sample_count": 1812, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451807, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:01:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T11:01:12+09:00", "last_frame_read_age_sec": 0.165, "last_frame_read_at": "2026-07-01T11:01:12+09:00", "mjpeg": {"age_sec": 0.158, "exists": true, "mtime": "2026-07-01T11:01:12.570690+09:00", "mtime_epoch": 1782871272.5706902, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171530}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:01:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5058.031, "name": "celery", "pid": 267, "rss_mb": 5058.031, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 171, "pushed_frames": 171, "read_frames": 170, "select_timeouts": 2, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.724, "pushed": 5.724, "read": 5.691}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.692, "count": 171, "max_ms": 41.254, "p95_ms": 35.721}, "gstreamer_push": {"avg_ms": 1.04, "count": 171, "max_ms": 1.308, "p95_ms": 1.144}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.433, "count": 171, "max_ms": 47.459, "p95_ms": 35.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.766, "count": 171, "max_ms": 89.809, "p95_ms": 87.29}, "yolo11_640_fire_11class": {"avg_ms": 48.165, "count": 171, "max_ms": 68.298, "p95_ms": 62.997}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.859, "count": 171, "max_ms": 85.88, "p95_ms": 82.468}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.376, "count": 171, "max_ms": 187.068, "p95_ms": 180.153}, "read_interval": {"avg_ms": 175.66, "count": 170, "max_ms": 2543.167, "p95_ms": 192.136}, "triton_wait": {"avg_ms": 65.995, "count": 171, "max_ms": 89.822, "p95_ms": 87.302}, "visualize": {"avg_ms": 4.808, "count": 171, "max_ms": 6.146, "p95_ms": 5.509}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12098}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11102.445, "limit_mb": 8796093022207.996}, "detail": {"pid": 99023, "restart_count": 554, "start_count": 555}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 505, "h264_decode_error_count": 1457, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:01:12+09:00", "last_error_summary": "[h264 @ 0x558940aa8a80] error while decoding MB 24 25, bytestream -12", "last_exit_at": "2026-07-01T11:01:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:01:15+09:00", "pid": 99023, "process": {"exists": true, "fd_count": 5, "hwm_mb": 135.926, "name": "ffmpeg", "pid": 99023, "rss_mb": 135.926, "state": "S (sleeping)", "threads": 106}, "restart_count": 554, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 555}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.299, "sample_count": 1786, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1451807, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:01:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.387, "last_frame_pushed_at": "2026-07-01T11:01:12+09:00", "last_frame_read_age_sec": 4.543, "last_frame_read_at": "2026-07-01T11:01:12+09:00", "mjpeg": {"age_sec": 4.369, "exists": true, "mtime": "2026-07-01T11:01:12.738692+09:00", "mtime_epoch": 1782871272.7386918, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170548}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:01:17+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5058.031, "name": "celery", "pid": 267, "rss_mb": 5058.031, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 2, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.878, "pushed": 4.878, "read": 4.845}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.443, "count": 146, "max_ms": 41.254, "p95_ms": 35.17}, "gstreamer_push": {"avg_ms": 1.048, "count": 146, "max_ms": 1.308, "p95_ms": 1.146}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.509, "count": 146, "max_ms": 47.459, "p95_ms": 35.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.835, "count": 146, "max_ms": 89.809, "p95_ms": 87.046}, "yolo11_640_fire_11class": {"avg_ms": 48.253, "count": 146, "max_ms": 68.298, "p95_ms": 62.857}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.049, "count": 146, "max_ms": 85.88, "p95_ms": 82.21}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.392, "count": 146, "max_ms": 185.446, "p95_ms": 177.723}, "read_interval": {"avg_ms": 176.17, "count": 145, "max_ms": 2543.167, "p95_ms": 190.23}, "triton_wait": {"avg_ms": 66.1, "count": 146, "max_ms": 89.822, "p95_ms": 87.057}, "visualize": {"avg_ms": 4.73, "count": 146, "max_ms": 6.008, "p95_ms": 5.481}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12103}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10999.441, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d840da8b00] left block unavailable for requested intra mode [h264 @ 0x55d840da8b00] error while decoding MB 0 32, bytestream 2545 [h264 @ 0x55d840cd9c80] error while decoding MB 33 23, bytestream -34", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 506, "h264_decode_error_count": 1459, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:03:07+09:00", "last_error_summary": "[h264 @ 0x55d840da8b00] left block unavailable for requested intra mode [h264 @ 0x55d840da8b00] error while decoding MB 0 32, bytestream 2545 [h264 @ 0x55d840cd9c80] error while decoding MB 33 23, bytestream -34", "last_exit_at": "2026-07-01T11:01:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.46, "last_started_at": "2026-07-01T11:01:15+09:00", "pid": 99023, "process": {"exists": false, "pid": 99023}, "restart_count": 554, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 555}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.098, "sample_count": 1675, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1452377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:03:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.098, "last_frame_pushed_at": "2026-07-01T11:03:07+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-01T11:03:07+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-01T11:03:07.483987+09:00", "mtime_epoch": 1782871387.4839866, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172137}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:03:07+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5058.293, "name": "celery", "pid": 267, "rss_mb": 5058.293, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 96, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.607, "pushed": 3.607, "read": 3.645}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.714, "count": 95, "max_ms": 46.526, "p95_ms": 42.95}, "gstreamer_push": {"avg_ms": 1.076, "count": 95, "max_ms": 1.316, "p95_ms": 1.196}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.396, "count": 95, "max_ms": 43.267, "p95_ms": 35.562}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.868, "count": 95, "max_ms": 90.312, "p95_ms": 86.22}, "yolo11_640_fire_11class": {"avg_ms": 48.234, "count": 95, "max_ms": 64.342, "p95_ms": 62.399}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.364, "count": 95, "max_ms": 82.87, "p95_ms": 81.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.038, "count": 95, "max_ms": 185.585, "p95_ms": 166.781}, "read_interval": {"avg_ms": 313.942, "count": 96, "max_ms": 8766.665, "p95_ms": 195.302}, "triton_wait": {"avg_ms": 64.977, "count": 95, "max_ms": 90.324, "p95_ms": 86.232}, "visualize": {"avg_ms": 4.392, "count": 95, "max_ms": 6.879, "p95_ms": 6.117}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 26.34, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12103}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10998.812, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:03:07+09:00", "last_error_summary": "[h264 @ 0x55d840da8b00] left block unavailable for requested intra mode [h264 @ 0x55d840da8b00] error while decoding MB 0 32, bytestream 2545 [h264 @ 0x55d840cd9c80] error while decoding MB 33 23, bytestream -34", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 506, "h264_decode_error_count": 1459, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:03:07+09:00", "last_error_summary": "[h264 @ 0x55d840da8b00] left block unavailable for requested intra mode [h264 @ 0x55d840da8b00] error while decoding MB 0 32, bytestream 2545 [h264 @ 0x55d840cd9c80] error while decoding MB 33 23, bytestream -34", "last_exit_at": "2026-07-01T11:03:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.528, "last_started_at": "2026-07-01T11:01:15+09:00", "pid": 99023, "process": {"exists": false, "pid": 99023}, "restart_count": 554, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 555}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.098, "sample_count": 1676, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1452378, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:03:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T11:03:07+09:00", "last_frame_read_age_sec": 0.157, "last_frame_read_at": "2026-07-01T11:03:07+09:00", "mjpeg": {"age_sec": 0.15, "exists": true, "mtime": "2026-07-01T11:03:07.483987+09:00", "mtime_epoch": 1782871387.4839866, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172137}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:03:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5058.293, "name": "celery", "pid": 267, "rss_mb": 5058.293, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.635, "pushed": 3.635, "read": 3.635}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.733, "count": 96, "max_ms": 46.526, "p95_ms": 42.95}, "gstreamer_push": {"avg_ms": 1.078, "count": 96, "max_ms": 1.316, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.384, "count": 96, "max_ms": 43.267, "p95_ms": 35.562}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.834, "count": 96, "max_ms": 90.312, "p95_ms": 86.22}, "yolo11_640_fire_11class": {"avg_ms": 48.218, "count": 96, "max_ms": 64.342, "p95_ms": 62.399}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.323, "count": 96, "max_ms": 82.87, "p95_ms": 81.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.136, "count": 96, "max_ms": 185.585, "p95_ms": 166.781}, "read_interval": {"avg_ms": 313.942, "count": 96, "max_ms": 8766.665, "p95_ms": 195.302}, "triton_wait": {"avg_ms": 64.942, "count": 96, "max_ms": 90.324, "p95_ms": 86.232}, "visualize": {"avg_ms": 4.419, "count": 96, "max_ms": 7.01, "p95_ms": 6.143}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 26.407, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12103}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10998.922, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 506, "h264_decode_error_count": 1459, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:03:07+09:00", "last_error_summary": "[h264 @ 0x55d840da8b00] left block unavailable for requested intra mode [h264 @ 0x55d840da8b00] error while decoding MB 0 32, bytestream 2545 [h264 @ 0x55d840cd9c80] error while decoding MB 33 23, bytestream -34", "last_exit_at": "2026-07-01T11:03:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.533, "last_started_at": "2026-07-01T11:01:15+09:00", "pid": 99023, "process": {"exists": false, "pid": 99023}, "restart_count": 554, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 555}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.098, "sample_count": 1676, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1452378, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:03:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T11:03:07+09:00", "last_frame_read_age_sec": 0.162, "last_frame_read_at": "2026-07-01T11:03:07+09:00", "mjpeg": {"age_sec": 0.156, "exists": true, "mtime": "2026-07-01T11:03:07.483987+09:00", "mtime_epoch": 1782871387.4839866, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172137}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:03:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5058.293, "name": "celery", "pid": 267, "rss_mb": 5058.293, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.635, "pushed": 3.635, "read": 3.635}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.733, "count": 96, "max_ms": 46.526, "p95_ms": 42.95}, "gstreamer_push": {"avg_ms": 1.078, "count": 96, "max_ms": 1.316, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.384, "count": 96, "max_ms": 43.267, "p95_ms": 35.562}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.834, "count": 96, "max_ms": 90.312, "p95_ms": 86.22}, "yolo11_640_fire_11class": {"avg_ms": 48.218, "count": 96, "max_ms": 64.342, "p95_ms": 62.399}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.323, "count": 96, "max_ms": 82.87, "p95_ms": 81.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.136, "count": 96, "max_ms": 185.585, "p95_ms": 166.781}, "read_interval": {"avg_ms": 313.942, "count": 96, "max_ms": 8766.665, "p95_ms": 195.302}, "triton_wait": {"avg_ms": 64.942, "count": 96, "max_ms": 90.324, "p95_ms": 86.232}, "visualize": {"avg_ms": 4.419, "count": 96, "max_ms": 7.01, "p95_ms": 6.143}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 26.412, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12103}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11102.316, "limit_mb": 8796093022207.996}, "detail": {"pid": 99130, "restart_count": 555, "start_count": 556}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 506, "h264_decode_error_count": 1459, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:03:07+09:00", "last_error_summary": "[h264 @ 0x55d840da8b00] left block unavailable for requested intra mode [h264 @ 0x55d840da8b00] error while decoding MB 0 32, bytestream 2545 [h264 @ 0x55d840cd9c80] error while decoding MB 33 23, bytestream -34", "last_exit_at": "2026-07-01T11:03:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:03:10+09:00", "pid": 99130, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.574, "name": "ffmpeg", "pid": 99130, "rss_mb": 133.574, "state": "S (sleeping)", "threads": 106}, "restart_count": 555, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 556}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.169, "sample_count": 1644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1452378, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:03:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.624, "last_frame_pushed_at": "2026-07-01T11:03:07+09:00", "last_frame_read_age_sec": 4.777, "last_frame_read_at": "2026-07-01T11:03:07+09:00", "mjpeg": {"age_sec": 4.61, "exists": true, "mtime": "2026-07-01T11:03:07.643988+09:00", "mtime_epoch": 1782871387.6439884, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169666}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:03:12+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5058.293, "name": "celery", "pid": 267, "rss_mb": 5058.293, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 90, "select_timeouts": 11, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.338, "pushed": 3.338, "read": 3.338}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.878, "count": 90, "max_ms": 46.526, "p95_ms": 42.95}, "gstreamer_push": {"avg_ms": 1.076, "count": 90, "max_ms": 1.316, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.374, "count": 90, "max_ms": 43.267, "p95_ms": 35.562}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.361, "count": 90, "max_ms": 90.312, "p95_ms": 86.082}, "yolo11_640_fire_11class": {"avg_ms": 48.112, "count": 90, "max_ms": 64.342, "p95_ms": 62.399}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.866, "count": 90, "max_ms": 82.87, "p95_ms": 81.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.053, "count": 90, "max_ms": 185.585, "p95_ms": 166.781}, "read_interval": {"avg_ms": 283.088, "count": 90, "max_ms": 8766.665, "p95_ms": 185.219}, "triton_wait": {"avg_ms": 64.476, "count": 90, "max_ms": 90.324, "p95_ms": 86.094}, "visualize": {"avg_ms": 4.429, "count": 90, "max_ms": 7.01, "p95_ms": 6.143}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 26.959, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 77, "tracking_summary": 13837}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10936.297, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5617a4278600] error while decoding MB 55 23, bytestream -5 [h264 @ 0x5617a4278600] error while decoding MB 48 47, bytestream -35 [in#0/rtsp @ 0x5617a4118e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5617a4118e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 215, "h264_decode_error_count": 222, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:04:13+09:00", "last_error_summary": "[h264 @ 0x5617a4278600] error while decoding MB 55 23, bytestream -5 [h264 @ 0x5617a4278600] error while decoding MB 48 47, bytestream -35 [in#0/rtsp @ 0x5617a4118e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5617a4118e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T10:41:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 1386.524, "last_started_at": "2026-07-01T10:41:07+09:00", "pid": 97779, "process": {"exists": false, "pid": 97779}, "restart_count": 214, "restart_count_per_hour": 5, "rtsp_timeout_count": 354, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 215}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.569, "sample_count": 2620, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1660448, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:04:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-01T11:04:13+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T11:04:13+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T11:04:13.740789+09:00", "mtime_epoch": 1782871453.7407887, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 213052}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:04:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1911.34, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.657, "pushed": 0.657, "read": 0.712}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 19.893, "count": 12, "max_ms": 23.678, "p95_ms": 23.678}, "gstreamer_push": {"avg_ms": 1.176, "count": 12, "max_ms": 1.329, "p95_ms": 1.329}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.494, "count": 12, "max_ms": 53.796, "p95_ms": 53.796}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.362, "count": 12, "max_ms": 60.695, "p95_ms": 60.695}, "yolo11_640_fire_11class": {"avg_ms": 46.016, "count": 12, "max_ms": 52.035, "p95_ms": 52.035}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.854, "count": 12, "max_ms": 55.527, "p95_ms": 55.527}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.19, "count": 12, "max_ms": 88.27, "p95_ms": 88.27}, "read_interval": {"avg_ms": 2747.954, "count": 13, "max_ms": 17536.297, "p95_ms": 17536.297}, "triton_wait": {"avg_ms": 52.374, "count": 12, "max_ms": 60.707, "p95_ms": 60.707}, "visualize": {"avg_ms": 3.311, "count": 12, "max_ms": 3.443, "p95_ms": 3.443}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.257, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 77, "tracking_summary": 13837}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10936.094, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:04:13+09:00", "last_error_summary": "[h264 @ 0x5617a4278600] error while decoding MB 55 23, bytestream -5 [h264 @ 0x5617a4278600] error while decoding MB 48 47, bytestream -35 [in#0/rtsp @ 0x5617a4118e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5617a4118e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 215, "h264_decode_error_count": 222, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:04:13+09:00", "last_error_summary": "[h264 @ 0x5617a4278600] error while decoding MB 55 23, bytestream -5 [h264 @ 0x5617a4278600] error while decoding MB 48 47, bytestream -35 [in#0/rtsp @ 0x5617a4118e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5617a4118e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:04:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 1386.551, "last_started_at": "2026-07-01T10:41:07+09:00", "pid": 97779, "process": {"exists": false, "pid": 97779}, "restart_count": 214, "restart_count_per_hour": 5, "rtsp_timeout_count": 354, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 215}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.569, "sample_count": 2621, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1660449, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:04:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T11:04:13+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-01T11:04:13+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-01T11:04:13.740789+09:00", "mtime_epoch": 1782871453.7407887, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 213052}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:04:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1911.34, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.711, "pushed": 0.711, "read": 0.711}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.094, "count": 13, "max_ms": 23.678, "p95_ms": 23.678}, "gstreamer_push": {"avg_ms": 1.193, "count": 13, "max_ms": 1.395, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.668, "count": 13, "max_ms": 53.796, "p95_ms": 53.796}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.309, "count": 13, "max_ms": 60.695, "p95_ms": 60.695}, "yolo11_640_fire_11class": {"avg_ms": 46.01, "count": 13, "max_ms": 52.035, "p95_ms": 52.035}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.842, "count": 13, "max_ms": 55.527, "p95_ms": 55.527}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.524, "count": 13, "max_ms": 88.27, "p95_ms": 88.27}, "read_interval": {"avg_ms": 2747.954, "count": 13, "max_ms": 17536.297, "p95_ms": 17536.297}, "triton_wait": {"avg_ms": 52.322, "count": 13, "max_ms": 60.707, "p95_ms": 60.707}, "visualize": {"avg_ms": 3.47, "count": 13, "max_ms": 5.379, "p95_ms": 5.379}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.283, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 77, "tracking_summary": 13837}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10936.168, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 215, "h264_decode_error_count": 222, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:04:13+09:00", "last_error_summary": "[h264 @ 0x5617a4278600] error while decoding MB 55 23, bytestream -5 [h264 @ 0x5617a4278600] error while decoding MB 48 47, bytestream -35 [in#0/rtsp @ 0x5617a4118e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5617a4118e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:04:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 1386.558, "last_started_at": "2026-07-01T10:41:07+09:00", "pid": 97779, "process": {"exists": false, "pid": 97779}, "restart_count": 214, "restart_count_per_hour": 5, "rtsp_timeout_count": 354, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 215}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.569, "sample_count": 2621, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1660449, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:04:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-01T11:04:13+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-01T11:04:13+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-01T11:04:13.740789+09:00", "mtime_epoch": 1782871453.7407887, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 213052}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:04:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1911.34, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.711, "pushed": 0.711, "read": 0.711}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.094, "count": 13, "max_ms": 23.678, "p95_ms": 23.678}, "gstreamer_push": {"avg_ms": 1.193, "count": 13, "max_ms": 1.395, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.668, "count": 13, "max_ms": 53.796, "p95_ms": 53.796}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.309, "count": 13, "max_ms": 60.695, "p95_ms": 60.695}, "yolo11_640_fire_11class": {"avg_ms": 46.01, "count": 13, "max_ms": 52.035, "p95_ms": 52.035}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.842, "count": 13, "max_ms": 55.527, "p95_ms": 55.527}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.524, "count": 13, "max_ms": 88.27, "p95_ms": 88.27}, "read_interval": {"avg_ms": 2747.954, "count": 13, "max_ms": 17536.297, "p95_ms": 17536.297}, "triton_wait": {"avg_ms": 52.322, "count": 13, "max_ms": 60.707, "p95_ms": 60.707}, "visualize": {"avg_ms": 3.47, "count": 13, "max_ms": 5.379, "p95_ms": 5.379}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.29, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 77, "tracking_summary": 13837}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 11113.617, "limit_mb": 8796093022207.996}, "detail": {"pid": 99237, "restart_count": 215, "start_count": 216}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 215, "h264_decode_error_count": 222, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:04:13+09:00", "last_error_summary": "[h264 @ 0x5617a4278600] error while decoding MB 55 23, bytestream -5 [h264 @ 0x5617a4278600] error while decoding MB 48 47, bytestream -35 [in#0/rtsp @ 0x5617a4118e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5617a4118e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:04:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:04:16+09:00", "pid": 99237, "process": {"exists": true, "fd_count": 5, "hwm_mb": 207.164, "name": "ffmpeg", "pid": 99237, "rss_mb": 207.164, "state": "S (sleeping)", "threads": 106}, "restart_count": 215, "restart_count_per_hour": 6, "rtsp_timeout_count": 354, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 216}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.553, "sample_count": 2574, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1660449, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:04:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.771, "last_frame_pushed_at": "2026-07-01T11:04:13+09:00", "last_frame_read_age_sec": 4.858, "last_frame_read_at": "2026-07-01T11:04:13+09:00", "mjpeg": {"age_sec": 4.755, "exists": true, "mtime": "2026-07-01T11:04:13.836790+09:00", "mtime_epoch": 1782871453.8367898, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 213019}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:04:18+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1911.34, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.564, "pushed": 0.564, "read": 0.564}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.094, "count": 13, "max_ms": 23.678, "p95_ms": 23.678}, "gstreamer_push": {"avg_ms": 1.193, "count": 13, "max_ms": 1.395, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.668, "count": 13, "max_ms": 53.796, "p95_ms": 53.796}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.309, "count": 13, "max_ms": 60.695, "p95_ms": 60.695}, "yolo11_640_fire_11class": {"avg_ms": 46.01, "count": 13, "max_ms": 52.035, "p95_ms": 52.035}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.842, "count": 13, "max_ms": 55.527, "p95_ms": 55.527}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.524, "count": 13, "max_ms": 88.27, "p95_ms": 88.27}, "read_interval": {"avg_ms": 2747.954, "count": 13, "max_ms": 17536.297, "p95_ms": 17536.297}, "triton_wait": {"avg_ms": 52.322, "count": 13, "max_ms": 60.707, "p95_ms": 60.707}, "visualize": {"avg_ms": 3.47, "count": 13, "max_ms": 5.379, "p95_ms": 5.379}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.045, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12112}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10935.445, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:03:07+09:00", "last_error_summary": "[h264 @ 0x55d840da8b00] left block unavailable for requested intra mode [h264 @ 0x55d840da8b00] error while decoding MB 0 32, bytestream 2545 [h264 @ 0x55d840cd9c80] error while decoding MB 33 23, bytestream -34", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 506, "h264_decode_error_count": 1459, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:03:07+09:00", "last_error_summary": "[h264 @ 0x55d840da8b00] left block unavailable for requested intra mode [h264 @ 0x55d840da8b00] error while decoding MB 0 32, bytestream 2545 [h264 @ 0x55d840cd9c80] error while decoding MB 33 23, bytestream -34", "last_exit_at": "2026-07-01T11:05:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.322, "last_started_at": "2026-07-01T11:03:10+09:00", "pid": 99130, "process": {"exists": false, "pid": 99130}, "restart_count": 555, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 556}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.994, "sample_count": 1738, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1453446, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:05:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T11:05:54+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T11:05:54+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-01T11:05:54.438072+09:00", "mtime_epoch": 1782871554.4380717, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168776}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:05:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5058.809, "name": "celery", "pid": 267, "rss_mb": 5058.809, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 188, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.267, "pushed": 6.267, "read": 6.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.549, "count": 188, "max_ms": 47.105, "p95_ms": 36.906}, "gstreamer_push": {"avg_ms": 1.083, "count": 188, "max_ms": 1.582, "p95_ms": 1.206}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.103, "count": 188, "max_ms": 43.119, "p95_ms": 36.05}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.067, "count": 188, "max_ms": 89.988, "p95_ms": 83.769}, "yolo11_640_fire_11class": {"avg_ms": 47.952, "count": 188, "max_ms": 68.835, "p95_ms": 62.208}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.082, "count": 188, "max_ms": 84.239, "p95_ms": 78.886}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.953, "count": 188, "max_ms": 181.006, "p95_ms": 167.055}, "read_interval": {"avg_ms": 159.716, "count": 188, "max_ms": 1635.414, "p95_ms": 178.582}, "triton_wait": {"avg_ms": 63.341, "count": 188, "max_ms": 90.0, "p95_ms": 84.222}, "visualize": {"avg_ms": 4.513, "count": 188, "max_ms": 5.648, "p95_ms": 5.472}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12112}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10935.43, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 506, "h264_decode_error_count": 1459, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:03:07+09:00", "last_error_summary": "[h264 @ 0x55d840da8b00] left block unavailable for requested intra mode [h264 @ 0x55d840da8b00] error while decoding MB 0 32, bytestream 2545 [h264 @ 0x55d840cd9c80] error while decoding MB 33 23, bytestream -34", "last_exit_at": "2026-07-01T11:05:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.326, "last_started_at": "2026-07-01T11:03:10+09:00", "pid": 99130, "process": {"exists": false, "pid": 99130}, "restart_count": 555, "restart_count_per_hour": 27, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 556}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.994, "sample_count": 1738, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1453446, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:05:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T11:05:54+09:00", "last_frame_read_age_sec": 0.147, "last_frame_read_at": "2026-07-01T11:05:54+09:00", "mjpeg": {"age_sec": 0.141, "exists": true, "mtime": "2026-07-01T11:05:54.438072+09:00", "mtime_epoch": 1782871554.4380717, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168776}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:05:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5058.809, "name": "celery", "pid": 267, "rss_mb": 5058.809, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 187, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.294, "pushed": 6.294, "read": 6.26}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.549, "count": 188, "max_ms": 47.105, "p95_ms": 36.906}, "gstreamer_push": {"avg_ms": 1.083, "count": 188, "max_ms": 1.582, "p95_ms": 1.206}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.103, "count": 188, "max_ms": 43.119, "p95_ms": 36.05}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.067, "count": 188, "max_ms": 89.988, "p95_ms": 83.769}, "yolo11_640_fire_11class": {"avg_ms": 47.952, "count": 188, "max_ms": 68.835, "p95_ms": 62.208}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.082, "count": 188, "max_ms": 84.239, "p95_ms": 78.886}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.953, "count": 188, "max_ms": 181.006, "p95_ms": 167.055}, "read_interval": {"avg_ms": 159.657, "count": 187, "max_ms": 1635.414, "p95_ms": 178.582}, "triton_wait": {"avg_ms": 63.341, "count": 188, "max_ms": 90.0, "p95_ms": 84.222}, "visualize": {"avg_ms": 4.513, "count": 188, "max_ms": 5.648, "p95_ms": 5.472}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12112}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11041.035, "limit_mb": 8796093022207.996}, "detail": {"pid": 99389, "restart_count": 556, "start_count": 557}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 506, "h264_decode_error_count": 1459, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:03:07+09:00", "last_error_summary": "[h264 @ 0x55d840da8b00] left block unavailable for requested intra mode [h264 @ 0x55d840da8b00] error while decoding MB 0 32, bytestream 2545 [h264 @ 0x55d840cd9c80] error while decoding MB 33 23, bytestream -34", "last_exit_at": "2026-07-01T11:05:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:05:57+09:00", "pid": 99389, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.516, "name": "ffmpeg", "pid": 99389, "rss_mb": 136.516, "state": "S (sleeping)", "threads": 106}, "restart_count": 556, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 557}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.199, "sample_count": 1718, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1453446, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:05:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.971, "last_frame_pushed_at": "2026-07-01T11:05:54+09:00", "last_frame_read_age_sec": 5.107, "last_frame_read_at": "2026-07-01T11:05:54+09:00", "mjpeg": {"age_sec": 4.954, "exists": true, "mtime": "2026-07-01T11:05:54.586074+09:00", "mtime_epoch": 1782871554.5860736, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168608}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:05:59+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5058.809, "name": "celery", "pid": 267, "rss_mb": 5058.809, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.186, "pushed": 5.186, "read": 5.152}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.638, "count": 155, "max_ms": 47.105, "p95_ms": 38.147}, "gstreamer_push": {"avg_ms": 1.083, "count": 155, "max_ms": 1.582, "p95_ms": 1.248}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.064, "count": 155, "max_ms": 43.119, "p95_ms": 36.818}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.501, "count": 155, "max_ms": 86.348, "p95_ms": 83.104}, "yolo11_640_fire_11class": {"avg_ms": 47.551, "count": 155, "max_ms": 68.835, "p95_ms": 62.136}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.548, "count": 155, "max_ms": 84.211, "p95_ms": 78.73}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.386, "count": 155, "max_ms": 181.006, "p95_ms": 167.994}, "read_interval": {"avg_ms": 161.787, "count": 154, "max_ms": 1635.414, "p95_ms": 181.973}, "triton_wait": {"avg_ms": 62.7, "count": 155, "max_ms": 86.362, "p95_ms": 83.426}, "visualize": {"avg_ms": 4.543, "count": 155, "max_ms": 5.648, "p95_ms": 5.472}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12119}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10937.941, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5587b25294c0] error while decoding MB 63 38, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 507, "h264_decode_error_count": 1460, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:08:25+09:00", "last_error_summary": "[h264 @ 0x5587b25294c0] error while decoding MB 63 38, bytestream -5", "last_exit_at": "2026-07-01T11:05:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.157, "last_started_at": "2026-07-01T11:05:57+09:00", "pid": 99389, "process": {"exists": false, "pid": 99389}, "restart_count": 556, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 557}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.423, "sample_count": 1894, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1454362, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:08:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T11:08:25+09:00", "last_frame_read_age_sec": 0.045, "last_frame_read_at": "2026-07-01T11:08:25+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T11:08:25.652116+09:00", "mtime_epoch": 1782871705.6521158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173326}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:08:25+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5059.324, "name": "celery", "pid": 267, "rss_mb": 5059.324, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 196, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.535, "pushed": 6.535, "read": 6.535}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.6, "count": 195, "max_ms": 50.692, "p95_ms": 35.705}, "gstreamer_push": {"avg_ms": 1.078, "count": 196, "max_ms": 1.317, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.736, "count": 195, "max_ms": 43.525, "p95_ms": 37.166}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.739, "count": 195, "max_ms": 91.672, "p95_ms": 85.68}, "yolo11_640_fire_11class": {"avg_ms": 48.766, "count": 195, "max_ms": 68.006, "p95_ms": 62.388}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.672, "count": 195, "max_ms": 86.678, "p95_ms": 80.26}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.922, "count": 196, "max_ms": 184.259, "p95_ms": 172.602}, "read_interval": {"avg_ms": 153.723, "count": 196, "max_ms": 194.718, "p95_ms": 183.114}, "triton_wait": {"avg_ms": 64.849, "count": 195, "max_ms": 91.684, "p95_ms": 85.866}, "visualize": {"avg_ms": 4.518, "count": 195, "max_ms": 6.972, "p95_ms": 5.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12119}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10937.164, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:08:25+09:00", "last_error_summary": "[h264 @ 0x5587b25294c0] error while decoding MB 63 38, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 507, "h264_decode_error_count": 1460, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:08:25+09:00", "last_error_summary": "[h264 @ 0x5587b25294c0] error while decoding MB 63 38, bytestream -5", "last_exit_at": "2026-07-01T11:08:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.249, "last_started_at": "2026-07-01T11:05:57+09:00", "pid": 99389, "process": {"exists": false, "pid": 99389}, "restart_count": 556, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 557}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.423, "sample_count": 1894, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1454363, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:08:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T11:08:25+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-01T11:08:25+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-01T11:08:25.652116+09:00", "mtime_epoch": 1782871705.6521158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173326}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:08:25+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5059.324, "name": "celery", "pid": 267, "rss_mb": 5059.324, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.551, "pushed": 6.551, "read": 6.517}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.597, "count": 196, "max_ms": 50.692, "p95_ms": 35.705}, "gstreamer_push": {"avg_ms": 1.079, "count": 196, "max_ms": 1.317, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.735, "count": 196, "max_ms": 43.525, "p95_ms": 37.166}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.719, "count": 196, "max_ms": 91.672, "p95_ms": 85.68}, "yolo11_640_fire_11class": {"avg_ms": 48.743, "count": 196, "max_ms": 68.006, "p95_ms": 62.388}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.63, "count": 196, "max_ms": 86.678, "p95_ms": 80.26}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.653, "count": 196, "max_ms": 184.259, "p95_ms": 169.679}, "read_interval": {"avg_ms": 153.513, "count": 195, "max_ms": 194.718, "p95_ms": 181.665}, "triton_wait": {"avg_ms": 64.828, "count": 196, "max_ms": 91.684, "p95_ms": 85.866}, "visualize": {"avg_ms": 4.517, "count": 196, "max_ms": 6.972, "p95_ms": 5.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12119}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10937.277, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 507, "h264_decode_error_count": 1460, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:08:25+09:00", "last_error_summary": "[h264 @ 0x5587b25294c0] error while decoding MB 63 38, bytestream -5", "last_exit_at": "2026-07-01T11:08:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.254, "last_started_at": "2026-07-01T11:05:57+09:00", "pid": 99389, "process": {"exists": false, "pid": 99389}, "restart_count": 556, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 557}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.423, "sample_count": 1894, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1454363, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:08:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T11:08:25+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T11:08:25+09:00", "mjpeg": {"age_sec": 0.141, "exists": true, "mtime": "2026-07-01T11:08:25.652116+09:00", "mtime_epoch": 1782871705.6521158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173326}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:08:25+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5059.324, "name": "celery", "pid": 267, "rss_mb": 5059.324, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.55, "pushed": 6.55, "read": 6.516}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.597, "count": 196, "max_ms": 50.692, "p95_ms": 35.705}, "gstreamer_push": {"avg_ms": 1.079, "count": 196, "max_ms": 1.317, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.735, "count": 196, "max_ms": 43.525, "p95_ms": 37.166}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.719, "count": 196, "max_ms": 91.672, "p95_ms": 85.68}, "yolo11_640_fire_11class": {"avg_ms": 48.743, "count": 196, "max_ms": 68.006, "p95_ms": 62.388}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.63, "count": 196, "max_ms": 86.678, "p95_ms": 80.26}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.653, "count": 196, "max_ms": 184.259, "p95_ms": 169.679}, "read_interval": {"avg_ms": 153.513, "count": 195, "max_ms": 194.718, "p95_ms": 181.665}, "triton_wait": {"avg_ms": 64.828, "count": 196, "max_ms": 91.684, "p95_ms": 85.866}, "visualize": {"avg_ms": 4.517, "count": 196, "max_ms": 6.972, "p95_ms": 5.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12119}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11044.863, "limit_mb": 8796093022207.996}, "detail": {"pid": 99496, "restart_count": 557, "start_count": 558}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 507, "h264_decode_error_count": 1460, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:08:25+09:00", "last_error_summary": "[h264 @ 0x5587b25294c0] error while decoding MB 63 38, bytestream -5", "last_exit_at": "2026-07-01T11:08:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:08:28+09:00", "pid": 99496, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.199, "name": "ffmpeg", "pid": 99496, "rss_mb": 139.199, "state": "S (sleeping)", "threads": 106}, "restart_count": 557, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 558}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.601, "sample_count": 1866, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1454363, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:08:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.245, "last_frame_pushed_at": "2026-07-01T11:08:25+09:00", "last_frame_read_age_sec": 4.378, "last_frame_read_at": "2026-07-01T11:08:25+09:00", "mjpeg": {"age_sec": 4.229, "exists": true, "mtime": "2026-07-01T11:08:25.800118+09:00", "mtime_epoch": 1782871705.800118, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173525}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:08:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5059.324, "name": "celery", "pid": 267, "rss_mb": 5059.324, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 168, "pushed_frames": 168, "read_frames": 167, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.616, "pushed": 5.616, "read": 5.583}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.737, "count": 168, "max_ms": 50.692, "p95_ms": 35.563}, "gstreamer_push": {"avg_ms": 1.08, "count": 168, "max_ms": 1.225, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.745, "count": 168, "max_ms": 43.525, "p95_ms": 37.237}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.654, "count": 168, "max_ms": 91.672, "p95_ms": 85.854}, "yolo11_640_fire_11class": {"avg_ms": 48.821, "count": 168, "max_ms": 68.006, "p95_ms": 62.2}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.677, "count": 168, "max_ms": 82.998, "p95_ms": 80.26}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.784, "count": 168, "max_ms": 184.259, "p95_ms": 169.679}, "read_interval": {"avg_ms": 153.675, "count": 167, "max_ms": 194.718, "p95_ms": 181.665}, "triton_wait": {"avg_ms": 64.772, "count": 168, "max_ms": 91.684, "p95_ms": 85.866}, "visualize": {"avg_ms": 4.53, "count": 168, "max_ms": 6.972, "p95_ms": 5.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10939.117, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5648f8980d00] error while decoding MB 59 10, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 508, "h264_decode_error_count": 1461, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:10:38+09:00", "last_error_summary": "[h264 @ 0x5648f8980d00] error while decoding MB 59 10, bytestream -7", "last_exit_at": "2026-07-01T11:08:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.576, "last_started_at": "2026-07-01T11:08:28+09:00", "pid": 99496, "process": {"exists": false, "pid": 99496}, "restart_count": 557, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 558}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.263, "sample_count": 1605, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1454947, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:10:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T11:10:38+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T11:10:38+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T11:10:38.558004+09:00", "mtime_epoch": 1782871838.5580037, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168081}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:10:38+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5059.582, "name": "celery", "pid": 267, "rss_mb": 5059.582, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 53, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.742, "pushed": 1.742, "read": 1.776}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.362, "count": 52, "max_ms": 41.535, "p95_ms": 37.908}, "gstreamer_push": {"avg_ms": 1.06, "count": 52, "max_ms": 1.24, "p95_ms": 1.14}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.694, "count": 52, "max_ms": 39.467, "p95_ms": 34.158}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.654, "count": 52, "max_ms": 82.462, "p95_ms": 82.23}, "yolo11_640_fire_11class": {"avg_ms": 49.005, "count": 52, "max_ms": 63.959, "p95_ms": 62.617}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.478, "count": 52, "max_ms": 79.742, "p95_ms": 78.714}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.868, "count": 52, "max_ms": 179.005, "p95_ms": 172.897}, "read_interval": {"avg_ms": 638.488, "count": 53, "max_ms": 9254.429, "p95_ms": 5217.129}, "triton_wait": {"avg_ms": 63.667, "count": 52, "max_ms": 82.475, "p95_ms": 82.243}, "visualize": {"avg_ms": 4.582, "count": 52, "max_ms": 5.621, "p95_ms": 5.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.844, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10938.363, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:10:38+09:00", "last_error_summary": "[h264 @ 0x5648f8980d00] error while decoding MB 59 10, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 508, "h264_decode_error_count": 1461, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:10:38+09:00", "last_error_summary": "[h264 @ 0x5648f8980d00] error while decoding MB 59 10, bytestream -7", "last_exit_at": "2026-07-01T11:10:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.661, "last_started_at": "2026-07-01T11:08:28+09:00", "pid": 99496, "process": {"exists": false, "pid": 99496}, "restart_count": 557, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 558}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.263, "sample_count": 1605, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1454948, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:10:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:10:38+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-01T11:10:38+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-01T11:10:38.558004+09:00", "mtime_epoch": 1782871838.5580037, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168081}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:10:38+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5059.582, "name": "celery", "pid": 267, "rss_mb": 5059.582, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 53, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.771, "pushed": 1.771, "read": 1.771}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.366, "count": 53, "max_ms": 41.535, "p95_ms": 37.908}, "gstreamer_push": {"avg_ms": 1.061, "count": 53, "max_ms": 1.24, "p95_ms": 1.14}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.729, "count": 53, "max_ms": 39.467, "p95_ms": 34.158}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.702, "count": 53, "max_ms": 82.462, "p95_ms": 82.23}, "yolo11_640_fire_11class": {"avg_ms": 49.009, "count": 53, "max_ms": 63.959, "p95_ms": 62.617}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.494, "count": 53, "max_ms": 79.742, "p95_ms": 78.714}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.67, "count": 53, "max_ms": 179.005, "p95_ms": 172.897}, "read_interval": {"avg_ms": 638.488, "count": 53, "max_ms": 9254.429, "p95_ms": 5217.129}, "triton_wait": {"avg_ms": 63.715, "count": 53, "max_ms": 82.475, "p95_ms": 82.243}, "visualize": {"avg_ms": 4.563, "count": 53, "max_ms": 5.621, "p95_ms": 5.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10938.383, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 508, "h264_decode_error_count": 1461, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:10:38+09:00", "last_error_summary": "[h264 @ 0x5648f8980d00] error while decoding MB 59 10, bytestream -7", "last_exit_at": "2026-07-01T11:10:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.665, "last_started_at": "2026-07-01T11:08:28+09:00", "pid": 99496, "process": {"exists": false, "pid": 99496}, "restart_count": 557, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 558}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.263, "sample_count": 1605, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1454948, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:10:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T11:10:38+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-01T11:10:38+09:00", "mjpeg": {"age_sec": 0.135, "exists": true, "mtime": "2026-07-01T11:10:38.558004+09:00", "mtime_epoch": 1782871838.5580037, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168081}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:10:38+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5059.582, "name": "celery", "pid": 267, "rss_mb": 5059.582, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 53, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.771, "pushed": 1.771, "read": 1.771}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.366, "count": 53, "max_ms": 41.535, "p95_ms": 37.908}, "gstreamer_push": {"avg_ms": 1.061, "count": 53, "max_ms": 1.24, "p95_ms": 1.14}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.729, "count": 53, "max_ms": 39.467, "p95_ms": 34.158}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.702, "count": 53, "max_ms": 82.462, "p95_ms": 82.23}, "yolo11_640_fire_11class": {"avg_ms": 49.009, "count": 53, "max_ms": 63.959, "p95_ms": 62.617}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.494, "count": 53, "max_ms": 79.742, "p95_ms": 78.714}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.67, "count": 53, "max_ms": 179.005, "p95_ms": 172.897}, "read_interval": {"avg_ms": 638.488, "count": 53, "max_ms": 9254.429, "p95_ms": 5217.129}, "triton_wait": {"avg_ms": 63.715, "count": 53, "max_ms": 82.475, "p95_ms": 82.243}, "visualize": {"avg_ms": 4.563, "count": 53, "max_ms": 5.621, "p95_ms": 5.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10948.953, "limit_mb": 8796093022207.996}, "detail": {"pid": 99603, "restart_count": 558, "start_count": 559}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 508, "h264_decode_error_count": 1461, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:10:38+09:00", "last_error_summary": "[h264 @ 0x5648f8980d00] error while decoding MB 59 10, bytestream -7", "last_exit_at": "2026-07-01T11:10:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:10:41+09:00", "pid": 99603, "process": {"exists": true, "fd_count": 4, "hwm_mb": 46.73, "name": "ffmpeg", "pid": 99603, "rss_mb": 46.73, "state": "S (sleeping)", "threads": 1}, "restart_count": 558, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 559}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 172.039, "sample_count": 1577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1454948, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:10:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.526, "last_frame_pushed_at": "2026-07-01T11:10:38+09:00", "last_frame_read_age_sec": 4.661, "last_frame_read_at": "2026-07-01T11:10:38+09:00", "mjpeg": {"age_sec": 4.503, "exists": true, "mtime": "2026-07-01T11:10:38.710006+09:00", "mtime_epoch": 1782871838.7100058, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168450}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:10:43+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5059.582, "name": "celery", "pid": 267, "rss_mb": 5059.582, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 35, "pushed_frames": 35, "read_frames": 34, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.169, "pushed": 1.169, "read": 1.135}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 33.699, "count": 35, "max_ms": 41.535, "p95_ms": 40.884}, "gstreamer_push": {"avg_ms": 1.07, "count": 35, "max_ms": 1.24, "p95_ms": 1.158}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.587, "count": 35, "max_ms": 39.467, "p95_ms": 35.85}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.555, "count": 35, "max_ms": 82.462, "p95_ms": 82.462}, "yolo11_640_fire_11class": {"avg_ms": 49.803, "count": 35, "max_ms": 63.959, "p95_ms": 62.784}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.445, "count": 35, "max_ms": 79.742, "p95_ms": 79.7}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.127, "count": 35, "max_ms": 177.313, "p95_ms": 172.897}, "read_interval": {"avg_ms": 748.402, "count": 34, "max_ms": 9254.429, "p95_ms": 5966.87}, "triton_wait": {"avg_ms": 64.567, "count": 35, "max_ms": 82.475, "p95_ms": 82.474}, "visualize": {"avg_ms": 4.43, "count": 35, "max_ms": 5.621, "p95_ms": 5.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10939.309, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560ca32efa00] error while decoding MB 65 21, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 509, "h264_decode_error_count": 1462, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:12:14+09:00", "last_error_summary": "[h264 @ 0x560ca32efa00] error while decoding MB 65 21, bytestream -5", "last_exit_at": "2026-07-01T11:10:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 93.321, "last_started_at": "2026-07-01T11:10:41+09:00", "pid": 99603, "process": {"exists": false, "pid": 99603}, "restart_count": 558, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 559}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.71, "sample_count": 1343, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1455256, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:12:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T11:12:14+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T11:12:14+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T11:12:14.487407+09:00", "mtime_epoch": 1782871934.4874072, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167165}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:12:14+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5059.84, "name": "celery", "pid": 267, "rss_mb": 5059.84, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.137, "pushed": 4.137, "read": 4.137}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.151, "count": 123, "max_ms": 38.846, "p95_ms": 35.016}, "gstreamer_push": {"avg_ms": 1.079, "count": 124, "max_ms": 1.608, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.582, "count": 123, "max_ms": 39.407, "p95_ms": 35.528}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.574, "count": 123, "max_ms": 88.528, "p95_ms": 84.392}, "yolo11_640_fire_11class": {"avg_ms": 48.674, "count": 123, "max_ms": 65.555, "p95_ms": 60.373}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.32, "count": 123, "max_ms": 83.583, "p95_ms": 79.074}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.115, "count": 124, "max_ms": 189.005, "p95_ms": 166.655}, "read_interval": {"avg_ms": 242.741, "count": 124, "max_ms": 3822.052, "p95_ms": 186.467}, "triton_wait": {"avg_ms": 64.616, "count": 123, "max_ms": 88.54, "p95_ms": 84.403}, "visualize": {"avg_ms": 4.454, "count": 123, "max_ms": 6.255, "p95_ms": 5.444}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10938.711, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:12:14+09:00", "last_error_summary": "[h264 @ 0x560ca32efa00] error while decoding MB 65 21, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 509, "h264_decode_error_count": 1462, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:12:14+09:00", "last_error_summary": "[h264 @ 0x560ca32efa00] error while decoding MB 65 21, bytestream -5", "last_exit_at": "2026-07-01T11:12:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 93.425, "last_started_at": "2026-07-01T11:10:41+09:00", "pid": 99603, "process": {"exists": false, "pid": 99603}, "restart_count": 558, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 559}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.71, "sample_count": 1343, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1455257, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:12:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T11:12:14+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-01T11:12:14+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-01T11:12:14.487407+09:00", "mtime_epoch": 1782871934.4874072, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167165}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:12:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5059.84, "name": "celery", "pid": 267, "rss_mb": 5059.84, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.145, "pushed": 4.145, "read": 4.111}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.135, "count": 124, "max_ms": 38.846, "p95_ms": 35.016}, "gstreamer_push": {"avg_ms": 1.08, "count": 124, "max_ms": 1.608, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.579, "count": 124, "max_ms": 39.407, "p95_ms": 35.528}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.705, "count": 124, "max_ms": 88.528, "p95_ms": 84.392}, "yolo11_640_fire_11class": {"avg_ms": 48.739, "count": 124, "max_ms": 65.555, "p95_ms": 60.373}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.368, "count": 124, "max_ms": 83.583, "p95_ms": 79.074}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.906, "count": 124, "max_ms": 189.005, "p95_ms": 166.004}, "read_interval": {"avg_ms": 243.198, "count": 123, "max_ms": 3822.052, "p95_ms": 184.416}, "triton_wait": {"avg_ms": 64.746, "count": 124, "max_ms": 88.54, "p95_ms": 84.403}, "visualize": {"avg_ms": 4.453, "count": 124, "max_ms": 6.255, "p95_ms": 5.444}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10938.68, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 509, "h264_decode_error_count": 1462, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:12:14+09:00", "last_error_summary": "[h264 @ 0x560ca32efa00] error while decoding MB 65 21, bytestream -5", "last_exit_at": "2026-07-01T11:12:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 93.43, "last_started_at": "2026-07-01T11:10:41+09:00", "pid": 99603, "process": {"exists": false, "pid": 99603}, "restart_count": 558, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 559}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.71, "sample_count": 1343, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1455257, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:12:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T11:12:14+09:00", "last_frame_read_age_sec": 0.161, "last_frame_read_at": "2026-07-01T11:12:14+09:00", "mjpeg": {"age_sec": 0.154, "exists": true, "mtime": "2026-07-01T11:12:14.487407+09:00", "mtime_epoch": 1782871934.4874072, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167165}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:12:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5059.84, "name": "celery", "pid": 267, "rss_mb": 5059.84, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.144, "pushed": 4.144, "read": 4.111}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.135, "count": 124, "max_ms": 38.846, "p95_ms": 35.016}, "gstreamer_push": {"avg_ms": 1.08, "count": 124, "max_ms": 1.608, "p95_ms": 1.145}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.579, "count": 124, "max_ms": 39.407, "p95_ms": 35.528}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.705, "count": 124, "max_ms": 88.528, "p95_ms": 84.392}, "yolo11_640_fire_11class": {"avg_ms": 48.739, "count": 124, "max_ms": 65.555, "p95_ms": 60.373}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.368, "count": 124, "max_ms": 83.583, "p95_ms": 79.074}}, "truncated_count": 0}, "processing_total": {"avg_ms": 139.906, "count": 124, "max_ms": 189.005, "p95_ms": 166.004}, "read_interval": {"avg_ms": 243.198, "count": 123, "max_ms": 3822.052, "p95_ms": 184.416}, "triton_wait": {"avg_ms": 64.746, "count": 124, "max_ms": 88.54, "p95_ms": 84.403}, "visualize": {"avg_ms": 4.453, "count": 124, "max_ms": 6.255, "p95_ms": 5.444}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12127}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11041.949, "limit_mb": 8796093022207.996}, "detail": {"pid": 99710, "restart_count": 559, "start_count": 560}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 509, "h264_decode_error_count": 1462, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:12:14+09:00", "last_error_summary": "[h264 @ 0x560ca32efa00] error while decoding MB 65 21, bytestream -5", "last_exit_at": "2026-07-01T11:12:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:12:17+09:00", "pid": 99710, "process": {"exists": true, "fd_count": 5, "hwm_mb": 134.07, "name": "ffmpeg", "pid": 99710, "rss_mb": 134.07, "state": "S (sleeping)", "threads": 106}, "restart_count": 559, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 560}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.028, "sample_count": 1315, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1455257, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:12:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.549, "last_frame_pushed_at": "2026-07-01T11:12:14+09:00", "last_frame_read_age_sec": 4.701, "last_frame_read_at": "2026-07-01T11:12:14+09:00", "mjpeg": {"age_sec": 4.53, "exists": true, "mtime": "2026-07-01T11:12:14.651410+09:00", "mtime_epoch": 1782871934.6514096, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164141}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:12:19+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5059.84, "name": "celery", "pid": 267, "rss_mb": 5059.84, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.418, "pushed": 3.418, "read": 3.418}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.279, "count": 96, "max_ms": 38.846, "p95_ms": 35.292}, "gstreamer_push": {"avg_ms": 1.078, "count": 96, "max_ms": 1.608, "p95_ms": 1.166}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.75, "count": 96, "max_ms": 39.407, "p95_ms": 35.56}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.861, "count": 96, "max_ms": 88.528, "p95_ms": 84.392}, "yolo11_640_fire_11class": {"avg_ms": 48.578, "count": 96, "max_ms": 62.705, "p95_ms": 60.086}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.0, "count": 96, "max_ms": 82.74, "p95_ms": 76.679}}, "truncated_count": 0}, "processing_total": {"avg_ms": 141.322, "count": 96, "max_ms": 189.005, "p95_ms": 169.53}, "read_interval": {"avg_ms": 270.392, "count": 96, "max_ms": 3822.052, "p95_ms": 294.429}, "triton_wait": {"avg_ms": 64.892, "count": 96, "max_ms": 88.54, "p95_ms": 84.403}, "visualize": {"avg_ms": 4.509, "count": 96, "max_ms": 6.255, "p95_ms": 5.451}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.089, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10940.504, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a4278420c0] error while decoding MB 20 22, bytestream -21", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 510, "h264_decode_error_count": 1463, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:14:05+09:00", "last_error_summary": "[h264 @ 0x55a4278420c0] error while decoding MB 20 22, bytestream -21", "last_exit_at": "2026-07-01T11:12:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.865, "last_started_at": "2026-07-01T11:12:17+09:00", "pid": 99710, "process": {"exists": false, "pid": 99710}, "restart_count": 559, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 560}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.454, "sample_count": 997, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1455587, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:14:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T11:14:04+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T11:14:04+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T11:14:04.997058+09:00", "mtime_epoch": 1782872044.9970584, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175043}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:14:05+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5059.84, "name": "celery", "pid": 267, "rss_mb": 5059.84, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.134, "pushed": 2.134, "read": 2.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.317, "count": 63, "max_ms": 37.01, "p95_ms": 35.42}, "gstreamer_push": {"avg_ms": 1.08, "count": 64, "max_ms": 1.156, "p95_ms": 1.149}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.42, "count": 63, "max_ms": 41.258, "p95_ms": 38.592}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.694, "count": 63, "max_ms": 90.223, "p95_ms": 85.289}, "yolo11_640_fire_11class": {"avg_ms": 49.063, "count": 63, "max_ms": 65.34, "p95_ms": 64.107}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.724, "count": 63, "max_ms": 82.005, "p95_ms": 81.129}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.805, "count": 64, "max_ms": 182.727, "p95_ms": 166.776}, "read_interval": {"avg_ms": 470.046, "count": 64, "max_ms": 6289.983, "p95_ms": 3215.475}, "triton_wait": {"avg_ms": 67.862, "count": 63, "max_ms": 90.235, "p95_ms": 85.302}, "visualize": {"avg_ms": 4.403, "count": 63, "max_ms": 5.521, "p95_ms": 5.351}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10939.785, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:14:05+09:00", "last_error_summary": "[h264 @ 0x55a4278420c0] error while decoding MB 20 22, bytestream -21", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 510, "h264_decode_error_count": 1463, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:14:05+09:00", "last_error_summary": "[h264 @ 0x55a4278420c0] error while decoding MB 20 22, bytestream -21", "last_exit_at": "2026-07-01T11:14:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.978, "last_started_at": "2026-07-01T11:12:17+09:00", "pid": 99710, "process": {"exists": false, "pid": 99710}, "restart_count": 559, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 560}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.454, "sample_count": 997, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1455588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:14:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:14:05+09:00", "last_frame_read_age_sec": 0.168, "last_frame_read_at": "2026-07-01T11:14:04+09:00", "mjpeg": {"age_sec": 0.162, "exists": true, "mtime": "2026-07-01T11:14:04.997058+09:00", "mtime_epoch": 1782872044.9970584, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175043}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:14:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5059.84, "name": "celery", "pid": 267, "rss_mb": 5059.84, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.137, "pushed": 2.137, "read": 2.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.321, "count": 64, "max_ms": 37.01, "p95_ms": 35.42}, "gstreamer_push": {"avg_ms": 1.079, "count": 64, "max_ms": 1.156, "p95_ms": 1.149}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.462, "count": 64, "max_ms": 41.258, "p95_ms": 38.592}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.697, "count": 64, "max_ms": 90.223, "p95_ms": 85.289}, "yolo11_640_fire_11class": {"avg_ms": 49.057, "count": 64, "max_ms": 65.34, "p95_ms": 64.107}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.746, "count": 64, "max_ms": 82.005, "p95_ms": 81.129}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.166, "count": 64, "max_ms": 182.727, "p95_ms": 166.776}, "read_interval": {"avg_ms": 475.086, "count": 63, "max_ms": 6289.983, "p95_ms": 3215.475}, "triton_wait": {"avg_ms": 67.864, "count": 64, "max_ms": 90.235, "p95_ms": 85.302}, "visualize": {"avg_ms": 4.417, "count": 64, "max_ms": 5.521, "p95_ms": 5.351}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10939.777, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 510, "h264_decode_error_count": 1463, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:14:05+09:00", "last_error_summary": "[h264 @ 0x55a4278420c0] error while decoding MB 20 22, bytestream -21", "last_exit_at": "2026-07-01T11:14:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.981, "last_started_at": "2026-07-01T11:12:17+09:00", "pid": 99710, "process": {"exists": false, "pid": 99710}, "restart_count": 559, "restart_count_per_hour": 28, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 560}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.454, "sample_count": 997, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1455588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:14:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T11:14:05+09:00", "last_frame_read_age_sec": 0.171, "last_frame_read_at": "2026-07-01T11:14:04+09:00", "mjpeg": {"age_sec": 0.165, "exists": true, "mtime": "2026-07-01T11:14:04.997058+09:00", "mtime_epoch": 1782872044.9970584, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175043}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:14:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5059.84, "name": "celery", "pid": 267, "rss_mb": 5059.84, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.136, "pushed": 2.136, "read": 2.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.321, "count": 64, "max_ms": 37.01, "p95_ms": 35.42}, "gstreamer_push": {"avg_ms": 1.079, "count": 64, "max_ms": 1.156, "p95_ms": 1.149}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.462, "count": 64, "max_ms": 41.258, "p95_ms": 38.592}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.697, "count": 64, "max_ms": 90.223, "p95_ms": 85.289}, "yolo11_640_fire_11class": {"avg_ms": 49.057, "count": 64, "max_ms": 65.34, "p95_ms": 64.107}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.746, "count": 64, "max_ms": 82.005, "p95_ms": 81.129}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.166, "count": 64, "max_ms": 182.727, "p95_ms": 166.776}, "read_interval": {"avg_ms": 475.086, "count": 63, "max_ms": 6289.983, "p95_ms": 3215.475}, "triton_wait": {"avg_ms": 67.864, "count": 64, "max_ms": 90.235, "p95_ms": 85.302}, "visualize": {"avg_ms": 4.417, "count": 64, "max_ms": 5.521, "p95_ms": 5.351}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12129}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10950.695, "limit_mb": 8796093022207.996}, "detail": {"pid": 99817, "restart_count": 560, "start_count": 561}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 510, "h264_decode_error_count": 1463, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:14:05+09:00", "last_error_summary": "[h264 @ 0x55a4278420c0] error while decoding MB 20 22, bytestream -21", "last_exit_at": "2026-07-01T11:14:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:14:07+09:00", "pid": 99817, "process": {"exists": true, "fd_count": 4, "hwm_mb": 46.934, "name": "ffmpeg", "pid": 99817, "rss_mb": 46.934, "state": "S (sleeping)", "threads": 1}, "restart_count": 560, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 561}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.574, "sample_count": 979, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1455588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:14:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.721, "last_frame_pushed_at": "2026-07-01T11:14:05+09:00", "last_frame_read_age_sec": 4.886, "last_frame_read_at": "2026-07-01T11:14:04+09:00", "mjpeg": {"age_sec": 4.704, "exists": true, "mtime": "2026-07-01T11:14:05.173061+09:00", "mtime_epoch": 1782872045.1730611, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175134}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:14:09+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5059.84, "name": "celery", "pid": 267, "rss_mb": 5059.84, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 41, "pushed_frames": 41, "read_frames": 40, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.37, "pushed": 1.37, "read": 1.337}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 32.328, "count": 41, "max_ms": 36.433, "p95_ms": 35.293}, "gstreamer_push": {"avg_ms": 1.071, "count": 41, "max_ms": 1.156, "p95_ms": 1.149}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.869, "count": 41, "max_ms": 41.258, "p95_ms": 39.125}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.977, "count": 41, "max_ms": 88.827, "p95_ms": 85.289}, "yolo11_640_fire_11class": {"avg_ms": 48.943, "count": 41, "max_ms": 65.34, "p95_ms": 64.107}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.576, "count": 41, "max_ms": 81.676, "p95_ms": 79.965}}, "truncated_count": 0}, "processing_total": {"avg_ms": 144.228, "count": 41, "max_ms": 174.713, "p95_ms": 166.776}, "read_interval": {"avg_ms": 629.063, "count": 40, "max_ms": 6289.983, "p95_ms": 3458.237}, "triton_wait": {"avg_ms": 67.14, "count": 41, "max_ms": 88.839, "p95_ms": 85.302}, "visualize": {"avg_ms": 4.475, "count": 41, "max_ms": 5.521, "p95_ms": 5.414}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 78, "tracking_summary": 13888}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10979.031, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55787e792a40] error while decoding MB 95 59, bytestream -13 [in#0/rtsp @ 0x55787e5d5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55787e5d5e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 216, "h264_decode_error_count": 223, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:15:28+09:00", "last_error_summary": "[h264 @ 0x55787e792a40] error while decoding MB 95 59, bytestream -13 [in#0/rtsp @ 0x55787e5d5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55787e5d5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:04:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 671.608, "last_started_at": "2026-07-01T11:04:16+09:00", "pid": 99237, "process": {"exists": false, "pid": 99237}, "restart_count": 215, "restart_count_per_hour": 5, "rtsp_timeout_count": 356, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 216}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.321, "sample_count": 2621, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1666667, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:15:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-01T11:15:28+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-01T11:15:28+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-01T11:15:28.146321+09:00", "mtime_epoch": 1782872128.146321, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217216}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:15:28+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1932.422, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.66, "pushed": 0.66, "read": 0.715}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 25.19, "count": 12, "max_ms": 27.344, "p95_ms": 27.344}, "gstreamer_push": {"avg_ms": 0.997, "count": 12, "max_ms": 1.056, "p95_ms": 1.056}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.188, "count": 12, "max_ms": 52.717, "p95_ms": 52.717}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.991, "count": 12, "max_ms": 66.868, "p95_ms": 66.868}, "yolo11_640_fire_11class": {"avg_ms": 45.233, "count": 12, "max_ms": 49.02, "p95_ms": 49.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.293, "count": 12, "max_ms": 68.289, "p95_ms": 68.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.385, "count": 12, "max_ms": 164.752, "p95_ms": 164.752}, "read_interval": {"avg_ms": 2316.016, "count": 13, "max_ms": 16439.445, "p95_ms": 16439.445}, "triton_wait": {"avg_ms": 56.376, "count": 12, "max_ms": 68.302, "p95_ms": 68.302}, "visualize": {"avg_ms": 7.807, "count": 12, "max_ms": 9.779, "p95_ms": 9.779}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.187, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 78, "tracking_summary": 13888}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10978.398, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:15:28+09:00", "last_error_summary": "[h264 @ 0x55787e792a40] error while decoding MB 95 59, bytestream -13 [in#0/rtsp @ 0x55787e5d5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55787e5d5e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 216, "h264_decode_error_count": 223, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:15:28+09:00", "last_error_summary": "[h264 @ 0x55787e792a40] error while decoding MB 95 59, bytestream -13 [in#0/rtsp @ 0x55787e5d5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55787e5d5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:15:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 671.687, "last_started_at": "2026-07-01T11:04:16+09:00", "pid": 99237, "process": {"exists": false, "pid": 99237}, "restart_count": 215, "restart_count_per_hour": 5, "rtsp_timeout_count": 356, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 216}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.321, "sample_count": 2621, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1666668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:15:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T11:15:28+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-01T11:15:28+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-01T11:15:28.146321+09:00", "mtime_epoch": 1782872128.146321, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217216}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:15:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1932.422, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.712, "pushed": 0.712, "read": 0.712}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 25.038, "count": 13, "max_ms": 27.344, "p95_ms": 27.344}, "gstreamer_push": {"avg_ms": 1.005, "count": 13, "max_ms": 1.104, "p95_ms": 1.104}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.622, "count": 13, "max_ms": 52.717, "p95_ms": 52.717}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.021, "count": 13, "max_ms": 69.378, "p95_ms": 69.378}, "yolo11_640_fire_11class": {"avg_ms": 45.315, "count": 13, "max_ms": 49.02, "p95_ms": 49.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.317, "count": 13, "max_ms": 68.604, "p95_ms": 68.604}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.549, "count": 13, "max_ms": 164.752, "p95_ms": 164.752}, "read_interval": {"avg_ms": 2316.016, "count": 13, "max_ms": 16439.445, "p95_ms": 16439.445}, "triton_wait": {"avg_ms": 57.377, "count": 13, "max_ms": 69.39, "p95_ms": 69.39}, "visualize": {"avg_ms": 7.563, "count": 13, "max_ms": 9.779, "p95_ms": 9.779}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.265, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 78, "tracking_summary": 13888}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10978.516, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 216, "h264_decode_error_count": 223, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:15:28+09:00", "last_error_summary": "[h264 @ 0x55787e792a40] error while decoding MB 95 59, bytestream -13 [in#0/rtsp @ 0x55787e5d5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55787e5d5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:15:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 671.691, "last_started_at": "2026-07-01T11:04:16+09:00", "pid": 99237, "process": {"exists": false, "pid": 99237}, "restart_count": 215, "restart_count_per_hour": 5, "rtsp_timeout_count": 356, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 216}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.321, "sample_count": 2621, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1666668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:15:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T11:15:28+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-01T11:15:28+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-01T11:15:28.146321+09:00", "mtime_epoch": 1782872128.146321, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217216}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:15:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1932.422, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.712, "pushed": 0.712, "read": 0.712}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 25.038, "count": 13, "max_ms": 27.344, "p95_ms": 27.344}, "gstreamer_push": {"avg_ms": 1.005, "count": 13, "max_ms": 1.104, "p95_ms": 1.104}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.622, "count": 13, "max_ms": 52.717, "p95_ms": 52.717}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.021, "count": 13, "max_ms": 69.378, "p95_ms": 69.378}, "yolo11_640_fire_11class": {"avg_ms": 45.315, "count": 13, "max_ms": 49.02, "p95_ms": 49.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.317, "count": 13, "max_ms": 68.604, "p95_ms": 68.604}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.549, "count": 13, "max_ms": 164.752, "p95_ms": 164.752}, "read_interval": {"avg_ms": 2316.016, "count": 13, "max_ms": 16439.445, "p95_ms": 16439.445}, "triton_wait": {"avg_ms": 57.377, "count": 13, "max_ms": 69.39, "p95_ms": 69.39}, "visualize": {"avg_ms": 7.563, "count": 13, "max_ms": 9.779, "p95_ms": 9.779}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.27, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 78, "tracking_summary": 13888}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 11026.977, "limit_mb": 8796093022207.996}, "detail": {"pid": 99984, "restart_count": 216, "start_count": 217}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 216, "h264_decode_error_count": 223, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:15:28+09:00", "last_error_summary": "[h264 @ 0x55787e792a40] error while decoding MB 95 59, bytestream -13 [in#0/rtsp @ 0x55787e5d5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55787e5d5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:15:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:15:31+09:00", "pid": 99984, "process": {"exists": true, "fd_count": 5, "hwm_mb": 82.328, "name": "ffmpeg", "pid": 99984, "rss_mb": 82.328, "state": "S (sleeping)", "threads": 19}, "restart_count": 216, "restart_count_per_hour": 6, "rtsp_timeout_count": 356, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 217}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.588, "sample_count": 2570, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1666668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:15:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.946, "last_frame_pushed_at": "2026-07-01T11:15:28+09:00", "last_frame_read_age_sec": 5.08, "last_frame_read_at": "2026-07-01T11:15:28+09:00", "mjpeg": {"age_sec": 4.929, "exists": true, "mtime": "2026-07-01T11:15:28.290323+09:00", "mtime_epoch": 1782872128.2903233, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:15:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1932.422, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.56, "pushed": 0.56, "read": 0.56}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 25.038, "count": 13, "max_ms": 27.344, "p95_ms": 27.344}, "gstreamer_push": {"avg_ms": 1.005, "count": 13, "max_ms": 1.104, "p95_ms": 1.104}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.622, "count": 13, "max_ms": 52.717, "p95_ms": 52.717}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.021, "count": 13, "max_ms": 69.378, "p95_ms": 69.378}, "yolo11_640_fire_11class": {"avg_ms": 45.315, "count": 13, "max_ms": 49.02, "p95_ms": 49.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.317, "count": 13, "max_ms": 68.604, "p95_ms": 68.604}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.549, "count": 13, "max_ms": 164.752, "p95_ms": 164.752}, "read_interval": {"avg_ms": 2316.016, "count": 13, "max_ms": 16439.445, "p95_ms": 16439.445}, "triton_wait": {"avg_ms": 57.377, "count": 13, "max_ms": 69.39, "p95_ms": 69.39}, "visualize": {"avg_ms": 7.563, "count": 13, "max_ms": 9.779, "p95_ms": 9.779}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.207, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11002.031, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b69066fb40] top block unavailable for requested intra mode -1 [h264 @ 0x55b69066fb40] error while decoding MB 56 0, bytestream 119223 [h264 @ 0x55b69073e9c0] error while decoding MB 63 32, bytestream -28", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 511, "h264_decode_error_count": 1465, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:15:53+09:00", "last_error_summary": "[h264 @ 0x55b69066fb40] top block unavailable for requested intra mode -1 [h264 @ 0x55b69066fb40] error while decoding MB 56 0, bytestream 119223 [h264 @ 0x55b69073e9c0] error while decoding MB 63 32, bytestream -28", "last_exit_at": "2026-07-01T11:14:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 105.534, "last_started_at": "2026-07-01T11:14:07+09:00", "pid": 99817, "process": {"exists": false, "pid": 99817}, "restart_count": 560, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 561}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.478, "sample_count": 1054, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1456002, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:15:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T11:15:53+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T11:15:53+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T11:15:53.366707+09:00", "mtime_epoch": 1782872153.3667068, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176447}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:15:53+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5060.098, "name": "celery", "pid": 267, "rss_mb": 5060.098, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 143, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.774, "pushed": 4.774, "read": 4.774}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.545, "count": 142, "max_ms": 47.026, "p95_ms": 39.318}, "gstreamer_push": {"avg_ms": 0.852, "count": 143, "max_ms": 0.938, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.173, "count": 142, "max_ms": 45.587, "p95_ms": 38.526}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.818, "count": 142, "max_ms": 93.406, "p95_ms": 85.843}, "yolo11_640_fire_11class": {"avg_ms": 42.681, "count": 142, "max_ms": 66.64, "p95_ms": 59.769}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.553, "count": 142, "max_ms": 90.144, "p95_ms": 83.442}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.096, "count": 143, "max_ms": 272.921, "p95_ms": 248.808}, "read_interval": {"avg_ms": 210.578, "count": 143, "max_ms": 283.238, "p95_ms": 258.832}, "triton_wait": {"avg_ms": 61.196, "count": 142, "max_ms": 93.418, "p95_ms": 85.854}, "visualize": {"avg_ms": 7.131, "count": 142, "max_ms": 9.43, "p95_ms": 8.509}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11001.207, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:15:53+09:00", "last_error_summary": "[h264 @ 0x55b69066fb40] top block unavailable for requested intra mode -1 [h264 @ 0x55b69066fb40] error while decoding MB 56 0, bytestream 119223 [h264 @ 0x55b69073e9c0] error while decoding MB 63 32, bytestream -28", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 511, "h264_decode_error_count": 1465, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:15:53+09:00", "last_error_summary": "[h264 @ 0x55b69066fb40] top block unavailable for requested intra mode -1 [h264 @ 0x55b69066fb40] error while decoding MB 56 0, bytestream 119223 [h264 @ 0x55b69073e9c0] error while decoding MB 63 32, bytestream -28", "last_exit_at": "2026-07-01T11:15:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 105.697, "last_started_at": "2026-07-01T11:14:07+09:00", "pid": 99817, "process": {"exists": false, "pid": 99817}, "restart_count": 560, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 561}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.478, "sample_count": 1055, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1456003, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:15:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:15:53+09:00", "last_frame_read_age_sec": 0.216, "last_frame_read_at": "2026-07-01T11:15:53+09:00", "mjpeg": {"age_sec": 0.207, "exists": true, "mtime": "2026-07-01T11:15:53.366707+09:00", "mtime_epoch": 1782872153.3667068, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176447}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:15:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5060.098, "name": "celery", "pid": 267, "rss_mb": 5060.098, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.782, "pushed": 4.782, "read": 4.748}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.568, "count": 142, "max_ms": 47.026, "p95_ms": 39.318}, "gstreamer_push": {"avg_ms": 0.852, "count": 143, "max_ms": 0.938, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.137, "count": 142, "max_ms": 45.587, "p95_ms": 38.526}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.929, "count": 142, "max_ms": 93.406, "p95_ms": 85.843}, "yolo11_640_fire_11class": {"avg_ms": 42.764, "count": 142, "max_ms": 66.64, "p95_ms": 59.769}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.663, "count": 142, "max_ms": 90.144, "p95_ms": 83.442}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.111, "count": 143, "max_ms": 272.921, "p95_ms": 248.808}, "read_interval": {"avg_ms": 210.525, "count": 142, "max_ms": 283.238, "p95_ms": 258.832}, "triton_wait": {"avg_ms": 61.308, "count": 142, "max_ms": 93.418, "p95_ms": 85.854}, "visualize": {"avg_ms": 7.124, "count": 142, "max_ms": 9.43, "p95_ms": 8.509}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11001.199, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 511, "h264_decode_error_count": 1465, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:15:53+09:00", "last_error_summary": "[h264 @ 0x55b69066fb40] top block unavailable for requested intra mode -1 [h264 @ 0x55b69066fb40] error while decoding MB 56 0, bytestream 119223 [h264 @ 0x55b69073e9c0] error while decoding MB 63 32, bytestream -28", "last_exit_at": "2026-07-01T11:15:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 105.703, "last_started_at": "2026-07-01T11:14:07+09:00", "pid": 99817, "process": {"exists": false, "pid": 99817}, "restart_count": 560, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 561}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.478, "sample_count": 1055, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1456003, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:15:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T11:15:53+09:00", "last_frame_read_age_sec": 0.222, "last_frame_read_at": "2026-07-01T11:15:53+09:00", "mjpeg": {"age_sec": 0.212, "exists": true, "mtime": "2026-07-01T11:15:53.366707+09:00", "mtime_epoch": 1782872153.3667068, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176447}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:15:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5060.098, "name": "celery", "pid": 267, "rss_mb": 5060.098, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.781, "pushed": 4.781, "read": 4.747}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.568, "count": 142, "max_ms": 47.026, "p95_ms": 39.318}, "gstreamer_push": {"avg_ms": 0.852, "count": 143, "max_ms": 0.938, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.137, "count": 142, "max_ms": 45.587, "p95_ms": 38.526}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.929, "count": 142, "max_ms": 93.406, "p95_ms": 85.843}, "yolo11_640_fire_11class": {"avg_ms": 42.764, "count": 142, "max_ms": 66.64, "p95_ms": 59.769}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.663, "count": 142, "max_ms": 90.144, "p95_ms": 83.442}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.111, "count": 143, "max_ms": 272.921, "p95_ms": 248.808}, "read_interval": {"avg_ms": 210.525, "count": 142, "max_ms": 283.238, "p95_ms": 258.832}, "triton_wait": {"avg_ms": 61.308, "count": 142, "max_ms": 93.418, "p95_ms": 85.854}, "visualize": {"avg_ms": 7.124, "count": 142, "max_ms": 9.43, "p95_ms": 8.509}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 440, "tracking_summary": 12133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11106.445, "limit_mb": 8796093022207.996}, "detail": {"pid": 100136, "restart_count": 561, "start_count": 562}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 511, "h264_decode_error_count": 1465, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:15:53+09:00", "last_error_summary": "[h264 @ 0x55b69066fb40] top block unavailable for requested intra mode -1 [h264 @ 0x55b69066fb40] error while decoding MB 56 0, bytestream 119223 [h264 @ 0x55b69073e9c0] error while decoding MB 63 32, bytestream -28", "last_exit_at": "2026-07-01T11:15:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:15:55+09:00", "pid": 100136, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.348, "name": "ffmpeg", "pid": 100136, "rss_mb": 136.348, "state": "S (sleeping)", "threads": 106}, "restart_count": 561, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 562}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.478, "sample_count": 1055, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1456003, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:15:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.071, "last_frame_pushed_at": "2026-07-01T11:15:53+09:00", "last_frame_read_age_sec": 4.284, "last_frame_read_at": "2026-07-01T11:15:53+09:00", "mjpeg": {"age_sec": 4.055, "exists": true, "mtime": "2026-07-01T11:15:53.586710+09:00", "mtime_epoch": 1782872153.5867102, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177334}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:15:57+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5060.098, "name": "celery", "pid": 267, "rss_mb": 5060.098, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.117, "pushed": 4.117, "read": 4.083}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 34.743, "count": 123, "max_ms": 47.026, "p95_ms": 39.676}, "gstreamer_push": {"avg_ms": 0.853, "count": 123, "max_ms": 0.934, "p95_ms": 0.903}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.24, "count": 123, "max_ms": 45.587, "p95_ms": 38.526}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.049, "count": 123, "max_ms": 93.406, "p95_ms": 86.275}, "yolo11_640_fire_11class": {"avg_ms": 43.287, "count": 123, "max_ms": 66.64, "p95_ms": 59.817}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.723, "count": 123, "max_ms": 90.144, "p95_ms": 83.924}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.81, "count": 123, "max_ms": 272.921, "p95_ms": 251.365}, "read_interval": {"avg_ms": 211.22, "count": 122, "max_ms": 283.238, "p95_ms": 260.653}, "triton_wait": {"avg_ms": 62.306, "count": 123, "max_ms": 93.418, "p95_ms": 86.288}, "visualize": {"avg_ms": 7.095, "count": 123, "max_ms": 9.43, "p95_ms": 8.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12136}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10956.945, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561e35d4b800] error while decoding MB 32 29, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 512, "h264_decode_error_count": 1466, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:17:54+09:00", "last_error_summary": "[h264 @ 0x561e35d4b800] error while decoding MB 32 29, bytestream -5", "last_exit_at": "2026-07-01T11:15:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.408, "last_started_at": "2026-07-01T11:15:55+09:00", "pid": 100136, "process": {"exists": false, "pid": 100136}, "restart_count": 561, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 562}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 331.994, "sample_count": 1000, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1456402, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:17:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T11:17:53+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T11:17:53+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T11:17:54.004569+09:00", "mtime_epoch": 1782872274.0045688, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:17:54+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5060.355, "name": "celery", "pid": 267, "rss_mb": 5060.355, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.47, "pushed": 3.47, "read": 3.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 40.074, "count": 103, "max_ms": 56.068, "p95_ms": 45.58}, "gstreamer_push": {"avg_ms": 0.855, "count": 103, "max_ms": 1.043, "p95_ms": 0.918}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.758, "count": 103, "max_ms": 69.006, "p95_ms": 51.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.54, "count": 103, "max_ms": 97.117, "p95_ms": 89.271}, "yolo11_640_fire_11class": {"avg_ms": 48.834, "count": 103, "max_ms": 74.657, "p95_ms": 63.417}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.625, "count": 103, "max_ms": 95.172, "p95_ms": 85.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 281.914, "count": 103, "max_ms": 388.16, "p95_ms": 351.853}, "read_interval": {"avg_ms": 291.017, "count": 103, "max_ms": 397.452, "p95_ms": 361.143}, "triton_wait": {"avg_ms": 70.315, "count": 103, "max_ms": 97.13, "p95_ms": 89.281}, "visualize": {"avg_ms": 12.933, "count": 103, "max_ms": 19.581, "p95_ms": 18.708}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.682, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12136}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10956.0, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:17:54+09:00", "last_error_summary": "[h264 @ 0x561e35d4b800] error while decoding MB 32 29, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 512, "h264_decode_error_count": 1466, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:17:54+09:00", "last_error_summary": "[h264 @ 0x561e35d4b800] error while decoding MB 32 29, bytestream -5", "last_exit_at": "2026-07-01T11:17:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.537, "last_started_at": "2026-07-01T11:15:55+09:00", "pid": 100136, "process": {"exists": false, "pid": 100136}, "restart_count": 561, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 562}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 331.994, "sample_count": 1001, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1456403, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:17:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:17:54+09:00", "last_frame_read_age_sec": 0.18, "last_frame_read_at": "2026-07-01T11:17:53+09:00", "mjpeg": {"age_sec": 0.173, "exists": true, "mtime": "2026-07-01T11:17:54.004569+09:00", "mtime_epoch": 1782872274.0045688, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:17:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5060.355, "name": "celery", "pid": 267, "rss_mb": 5060.355, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.489, "pushed": 3.489, "read": 3.455}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 40.045, "count": 103, "max_ms": 56.068, "p95_ms": 45.58}, "gstreamer_push": {"avg_ms": 0.855, "count": 104, "max_ms": 1.043, "p95_ms": 0.918}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.722, "count": 103, "max_ms": 69.006, "p95_ms": 51.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.503, "count": 103, "max_ms": 97.117, "p95_ms": 89.271}, "yolo11_640_fire_11class": {"avg_ms": 48.906, "count": 103, "max_ms": 74.657, "p95_ms": 63.417}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.575, "count": 103, "max_ms": 95.172, "p95_ms": 85.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 280.899, "count": 104, "max_ms": 388.16, "p95_ms": 351.853}, "read_interval": {"avg_ms": 291.017, "count": 103, "max_ms": 397.452, "p95_ms": 361.143}, "triton_wait": {"avg_ms": 70.264, "count": 103, "max_ms": 97.13, "p95_ms": 89.281}, "visualize": {"avg_ms": 12.854, "count": 103, "max_ms": 19.581, "p95_ms": 18.708}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.811, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12136}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10955.992, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 512, "h264_decode_error_count": 1466, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:17:54+09:00", "last_error_summary": "[h264 @ 0x561e35d4b800] error while decoding MB 32 29, bytestream -5", "last_exit_at": "2026-07-01T11:17:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.541, "last_started_at": "2026-07-01T11:15:55+09:00", "pid": 100136, "process": {"exists": false, "pid": 100136}, "restart_count": 561, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 562}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 331.994, "sample_count": 1001, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1456403, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:17:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T11:17:54+09:00", "last_frame_read_age_sec": 0.184, "last_frame_read_at": "2026-07-01T11:17:53+09:00", "mjpeg": {"age_sec": 0.176, "exists": true, "mtime": "2026-07-01T11:17:54.004569+09:00", "mtime_epoch": 1782872274.0045688, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:17:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5060.355, "name": "celery", "pid": 267, "rss_mb": 5060.355, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.488, "pushed": 3.488, "read": 3.455}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 40.045, "count": 103, "max_ms": 56.068, "p95_ms": 45.58}, "gstreamer_push": {"avg_ms": 0.855, "count": 104, "max_ms": 1.043, "p95_ms": 0.918}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.722, "count": 103, "max_ms": 69.006, "p95_ms": 51.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.503, "count": 103, "max_ms": 97.117, "p95_ms": 89.271}, "yolo11_640_fire_11class": {"avg_ms": 48.906, "count": 103, "max_ms": 74.657, "p95_ms": 63.417}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.575, "count": 103, "max_ms": 95.172, "p95_ms": 85.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 280.899, "count": 104, "max_ms": 388.16, "p95_ms": 351.853}, "read_interval": {"avg_ms": 291.017, "count": 103, "max_ms": 397.452, "p95_ms": 361.143}, "triton_wait": {"avg_ms": 70.264, "count": 103, "max_ms": 97.13, "p95_ms": 89.281}, "visualize": {"avg_ms": 12.854, "count": 103, "max_ms": 19.581, "p95_ms": 18.708}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.814, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12136}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11061.973, "limit_mb": 8796093022207.996}, "detail": {"pid": 100303, "restart_count": 562, "start_count": 563}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 512, "h264_decode_error_count": 1466, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:17:54+09:00", "last_error_summary": "[h264 @ 0x561e35d4b800] error while decoding MB 32 29, bytestream -5", "last_exit_at": "2026-07-01T11:17:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:17:57+09:00", "pid": 100303, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.156, "name": "ffmpeg", "pid": 100303, "rss_mb": 136.156, "state": "S (sleeping)", "threads": 106}, "restart_count": 562, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 563}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 332.089, "sample_count": 974, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1456403, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:17:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.88, "last_frame_pushed_at": "2026-07-01T11:17:54+09:00", "last_frame_read_age_sec": 5.058, "last_frame_read_at": "2026-07-01T11:17:53+09:00", "mjpeg": {"age_sec": 4.863, "exists": true, "mtime": "2026-07-01T11:17:54.192572+09:00", "mtime_epoch": 1782872274.1925719, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174371}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:17:59+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5060.355, "name": "celery", "pid": 267, "rss_mb": 5060.355, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 89, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.006, "pushed": 3.006, "read": 2.973}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 39.596, "count": 89, "max_ms": 56.068, "p95_ms": 44.629}, "gstreamer_push": {"avg_ms": 0.858, "count": 90, "max_ms": 1.043, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.681, "count": 89, "max_ms": 69.006, "p95_ms": 50.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.136, "count": 89, "max_ms": 97.117, "p95_ms": 88.853}, "yolo11_640_fire_11class": {"avg_ms": 49.169, "count": 89, "max_ms": 74.657, "p95_ms": 63.674}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.2, "count": 89, "max_ms": 95.172, "p95_ms": 85.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 273.186, "count": 90, "max_ms": 388.16, "p95_ms": 354.098}, "read_interval": {"avg_ms": 283.313, "count": 89, "max_ms": 397.452, "p95_ms": 365.118}, "triton_wait": {"avg_ms": 69.922, "count": 89, "max_ms": 97.13, "p95_ms": 88.866}, "visualize": {"avg_ms": 12.37, "count": 89, "max_ms": 19.581, "p95_ms": 18.708}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12141}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10958.066, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a48d829300] error while decoding MB 31 20, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 513, "h264_decode_error_count": 1467, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:20:05+09:00", "last_error_summary": "[h264 @ 0x55a48d829300] error while decoding MB 31 20, bytestream -13", "last_exit_at": "2026-07-01T11:17:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.346, "last_started_at": "2026-07-01T11:17:57+09:00", "pid": 100303, "process": {"exists": false, "pid": 100303}, "restart_count": 562, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 563}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 328.927, "sample_count": 1245, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1457013, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:20:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T11:20:05+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T11:20:05+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T11:20:05.354622+09:00", "mtime_epoch": 1782872405.3546221, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172649}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:20:05+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5060.613, "name": "celery", "pid": 267, "rss_mb": 5060.613, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.935, "pushed": 4.935, "read": 4.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.94, "count": 147, "max_ms": 48.02, "p95_ms": 43.459}, "gstreamer_push": {"avg_ms": 0.855, "count": 147, "max_ms": 1.054, "p95_ms": 0.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.113, "count": 147, "max_ms": 42.207, "p95_ms": 36.031}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.73, "count": 147, "max_ms": 92.49, "p95_ms": 85.015}, "yolo11_640_fire_11class": {"avg_ms": 46.91, "count": 147, "max_ms": 64.051, "p95_ms": 61.988}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.313, "count": 147, "max_ms": 88.142, "p95_ms": 83.664}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.055, "count": 147, "max_ms": 224.921, "p95_ms": 219.549}, "read_interval": {"avg_ms": 203.761, "count": 147, "max_ms": 235.328, "p95_ms": 228.558}, "triton_wait": {"avg_ms": 66.07, "count": 147, "max_ms": 92.502, "p95_ms": 86.065}, "visualize": {"avg_ms": 6.451, "count": 147, "max_ms": 8.108, "p95_ms": 7.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.788, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12141}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10957.324, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:20:05+09:00", "last_error_summary": "[h264 @ 0x55a48d829300] error while decoding MB 31 20, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 513, "h264_decode_error_count": 1467, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:20:05+09:00", "last_error_summary": "[h264 @ 0x55a48d829300] error while decoding MB 31 20, bytestream -13", "last_exit_at": "2026-07-01T11:20:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.498, "last_started_at": "2026-07-01T11:17:57+09:00", "pid": 100303, "process": {"exists": false, "pid": 100303}, "restart_count": 562, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 563}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 328.927, "sample_count": 1245, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1457014, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:20:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:20:05+09:00", "last_frame_read_age_sec": 0.206, "last_frame_read_at": "2026-07-01T11:20:05+09:00", "mjpeg": {"age_sec": 0.197, "exists": true, "mtime": "2026-07-01T11:20:05.354622+09:00", "mtime_epoch": 1782872405.3546221, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172649}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:20:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5060.613, "name": "celery", "pid": 267, "rss_mb": 5060.613, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.943, "pushed": 4.943, "read": 4.91}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.945, "count": 147, "max_ms": 48.02, "p95_ms": 43.459}, "gstreamer_push": {"avg_ms": 0.855, "count": 148, "max_ms": 1.054, "p95_ms": 0.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.108, "count": 147, "max_ms": 42.207, "p95_ms": 36.031}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.754, "count": 147, "max_ms": 92.49, "p95_ms": 85.015}, "yolo11_640_fire_11class": {"avg_ms": 46.956, "count": 147, "max_ms": 64.051, "p95_ms": 61.988}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.373, "count": 147, "max_ms": 88.142, "p95_ms": 83.664}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.105, "count": 148, "max_ms": 224.921, "p95_ms": 219.549}, "read_interval": {"avg_ms": 203.761, "count": 147, "max_ms": 235.328, "p95_ms": 228.558}, "triton_wait": {"avg_ms": 66.094, "count": 147, "max_ms": 92.502, "p95_ms": 86.065}, "visualize": {"avg_ms": 6.448, "count": 147, "max_ms": 8.108, "p95_ms": 7.269}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12141}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10957.438, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 513, "h264_decode_error_count": 1467, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:20:05+09:00", "last_error_summary": "[h264 @ 0x55a48d829300] error while decoding MB 31 20, bytestream -13", "last_exit_at": "2026-07-01T11:20:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.501, "last_started_at": "2026-07-01T11:17:57+09:00", "pid": 100303, "process": {"exists": false, "pid": 100303}, "restart_count": 562, "restart_count_per_hour": 29, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 563}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 328.927, "sample_count": 1245, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1457014, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:20:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T11:20:05+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T11:20:05+09:00", "mjpeg": {"age_sec": 0.201, "exists": true, "mtime": "2026-07-01T11:20:05.354622+09:00", "mtime_epoch": 1782872405.3546221, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172649}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:20:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5060.613, "name": "celery", "pid": 267, "rss_mb": 5060.613, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.943, "pushed": 4.943, "read": 4.909}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.945, "count": 147, "max_ms": 48.02, "p95_ms": 43.459}, "gstreamer_push": {"avg_ms": 0.855, "count": 148, "max_ms": 1.054, "p95_ms": 0.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.108, "count": 147, "max_ms": 42.207, "p95_ms": 36.031}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.754, "count": 147, "max_ms": 92.49, "p95_ms": 85.015}, "yolo11_640_fire_11class": {"avg_ms": 46.956, "count": 147, "max_ms": 64.051, "p95_ms": 61.988}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.373, "count": 147, "max_ms": 88.142, "p95_ms": 83.664}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.105, "count": 148, "max_ms": 224.921, "p95_ms": 219.549}, "read_interval": {"avg_ms": 203.761, "count": 147, "max_ms": 235.328, "p95_ms": 228.558}, "triton_wait": {"avg_ms": 66.094, "count": 147, "max_ms": 92.502, "p95_ms": 86.065}, "visualize": {"avg_ms": 6.448, "count": 147, "max_ms": 8.108, "p95_ms": 7.269}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12141}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11074.113, "limit_mb": 8796093022207.996}, "detail": {"pid": 100410, "restart_count": 563, "start_count": 564}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 513, "h264_decode_error_count": 1467, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:20:05+09:00", "last_error_summary": "[h264 @ 0x55a48d829300] error while decoding MB 31 20, bytestream -13", "last_exit_at": "2026-07-01T11:20:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:20:08+09:00", "pid": 100410, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.176, "name": "ffmpeg", "pid": 100410, "rss_mb": 147.176, "state": "S (sleeping)", "threads": 106}, "restart_count": 563, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 564}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 329.095, "sample_count": 1217, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1457014, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:20:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.937, "last_frame_pushed_at": "2026-07-01T11:20:05+09:00", "last_frame_read_age_sec": 5.14, "last_frame_read_at": "2026-07-01T11:20:05+09:00", "mjpeg": {"age_sec": 4.92, "exists": true, "mtime": "2026-07-01T11:20:05.566625+09:00", "mtime_epoch": 1782872405.5666254, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173075}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:20:10+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5060.613, "name": "celery", "pid": 267, "rss_mb": 5060.613, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.117, "pushed": 4.117, "read": 4.083}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.009, "count": 123, "max_ms": 48.02, "p95_ms": 43.676}, "gstreamer_push": {"avg_ms": 0.854, "count": 123, "max_ms": 1.054, "p95_ms": 0.919}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.044, "count": 123, "max_ms": 42.207, "p95_ms": 36.031}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.641, "count": 123, "max_ms": 88.688, "p95_ms": 84.997}, "yolo11_640_fire_11class": {"avg_ms": 47.034, "count": 123, "max_ms": 64.051, "p95_ms": 61.988}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.762, "count": 123, "max_ms": 87.045, "p95_ms": 83.565}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.523, "count": 123, "max_ms": 224.921, "p95_ms": 219.549}, "read_interval": {"avg_ms": 204.192, "count": 122, "max_ms": 235.328, "p95_ms": 228.143}, "triton_wait": {"avg_ms": 66.044, "count": 123, "max_ms": 88.699, "p95_ms": 85.649}, "visualize": {"avg_ms": 6.441, "count": 123, "max_ms": 8.096, "p95_ms": 7.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10959.336, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a4b4e41680] error while decoding MB 52 18, bytestream -26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 514, "h264_decode_error_count": 1468, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:22:11+09:00", "last_error_summary": "[h264 @ 0x55a4b4e41680] error while decoding MB 52 18, bytestream -26", "last_exit_at": "2026-07-01T11:20:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.027, "last_started_at": "2026-07-01T11:20:08+09:00", "pid": 100410, "process": {"exists": false, "pid": 100410}, "restart_count": 563, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 564}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 320.507, "sample_count": 1359, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1457622, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:22:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T11:22:11+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T11:22:11+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T11:22:11.468613+09:00", "mtime_epoch": 1782872531.4686131, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176206}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:22:11+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5060.871, "name": "celery", "pid": 267, "rss_mb": 5060.871, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 153, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.107, "pushed": 5.107, "read": 5.107}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.615, "count": 152, "max_ms": 46.526, "p95_ms": 42.047}, "gstreamer_push": {"avg_ms": 0.877, "count": 153, "max_ms": 1.184, "p95_ms": 0.951}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 28.882, "count": 152, "max_ms": 38.277, "p95_ms": 34.824}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.626, "count": 152, "max_ms": 89.727, "p95_ms": 79.125}, "yolo11_640_fire_11class": {"avg_ms": 45.375, "count": 152, "max_ms": 67.834, "p95_ms": 58.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.158, "count": 152, "max_ms": 87.014, "p95_ms": 82.362}}, "truncated_count": 0}, "processing_total": {"avg_ms": 186.907, "count": 153, "max_ms": 230.257, "p95_ms": 213.156}, "read_interval": {"avg_ms": 196.865, "count": 153, "max_ms": 239.559, "p95_ms": 224.595}, "triton_wait": {"avg_ms": 61.928, "count": 152, "max_ms": 89.737, "p95_ms": 82.374}, "visualize": {"avg_ms": 6.251, "count": 152, "max_ms": 8.671, "p95_ms": 7.212}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10958.547, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:22:11+09:00", "last_error_summary": "[h264 @ 0x55a4b4e41680] error while decoding MB 52 18, bytestream -26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 514, "h264_decode_error_count": 1468, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:22:11+09:00", "last_error_summary": "[h264 @ 0x55a4b4e41680] error while decoding MB 52 18, bytestream -26", "last_exit_at": "2026-07-01T11:22:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.158, "last_started_at": "2026-07-01T11:20:08+09:00", "pid": 100410, "process": {"exists": false, "pid": 100410}, "restart_count": 563, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 564}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 320.507, "sample_count": 1360, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1457623, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:22:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:22:11+09:00", "last_frame_read_age_sec": 0.182, "last_frame_read_at": "2026-07-01T11:22:11+09:00", "mjpeg": {"age_sec": 0.174, "exists": true, "mtime": "2026-07-01T11:22:11.468613+09:00", "mtime_epoch": 1782872531.4686131, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176206}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:22:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5060.871, "name": "celery", "pid": 267, "rss_mb": 5060.871, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.118, "pushed": 5.118, "read": 5.084}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.626, "count": 153, "max_ms": 46.526, "p95_ms": 42.047}, "gstreamer_push": {"avg_ms": 0.878, "count": 153, "max_ms": 1.184, "p95_ms": 0.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 28.842, "count": 153, "max_ms": 38.277, "p95_ms": 34.824}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.603, "count": 153, "max_ms": 89.727, "p95_ms": 79.125}, "yolo11_640_fire_11class": {"avg_ms": 45.36, "count": 153, "max_ms": 67.834, "p95_ms": 58.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.14, "count": 153, "max_ms": 87.014, "p95_ms": 82.362}}, "truncated_count": 0}, "processing_total": {"avg_ms": 186.688, "count": 153, "max_ms": 230.257, "p95_ms": 213.156}, "read_interval": {"avg_ms": 196.703, "count": 152, "max_ms": 239.559, "p95_ms": 224.595}, "triton_wait": {"avg_ms": 61.904, "count": 153, "max_ms": 89.737, "p95_ms": 82.374}, "visualize": {"avg_ms": 6.251, "count": 153, "max_ms": 8.671, "p95_ms": 7.212}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10958.66, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 514, "h264_decode_error_count": 1468, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:22:11+09:00", "last_error_summary": "[h264 @ 0x55a4b4e41680] error while decoding MB 52 18, bytestream -26", "last_exit_at": "2026-07-01T11:22:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.162, "last_started_at": "2026-07-01T11:20:08+09:00", "pid": 100410, "process": {"exists": false, "pid": 100410}, "restart_count": 563, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 564}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 320.507, "sample_count": 1360, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1457623, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:22:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T11:22:11+09:00", "last_frame_read_age_sec": 0.186, "last_frame_read_at": "2026-07-01T11:22:11+09:00", "mjpeg": {"age_sec": 0.178, "exists": true, "mtime": "2026-07-01T11:22:11.468613+09:00", "mtime_epoch": 1782872531.4686131, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176206}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:22:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5060.871, "name": "celery", "pid": 267, "rss_mb": 5060.871, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.117, "pushed": 5.117, "read": 5.084}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.626, "count": 153, "max_ms": 46.526, "p95_ms": 42.047}, "gstreamer_push": {"avg_ms": 0.878, "count": 153, "max_ms": 1.184, "p95_ms": 0.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 28.842, "count": 153, "max_ms": 38.277, "p95_ms": 34.824}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.603, "count": 153, "max_ms": 89.727, "p95_ms": 79.125}, "yolo11_640_fire_11class": {"avg_ms": 45.36, "count": 153, "max_ms": 67.834, "p95_ms": 58.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.14, "count": 153, "max_ms": 87.014, "p95_ms": 82.362}}, "truncated_count": 0}, "processing_total": {"avg_ms": 186.688, "count": 153, "max_ms": 230.257, "p95_ms": 213.156}, "read_interval": {"avg_ms": 196.703, "count": 152, "max_ms": 239.559, "p95_ms": 224.595}, "triton_wait": {"avg_ms": 61.904, "count": 153, "max_ms": 89.737, "p95_ms": 82.374}, "visualize": {"avg_ms": 6.251, "count": 153, "max_ms": 8.671, "p95_ms": 7.212}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12146}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11066.816, "limit_mb": 8796093022207.996}, "detail": {"pid": 100517, "restart_count": 564, "start_count": 565}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 514, "h264_decode_error_count": 1468, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:22:11+09:00", "last_error_summary": "[h264 @ 0x55a4b4e41680] error while decoding MB 52 18, bytestream -26", "last_exit_at": "2026-07-01T11:22:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:22:14+09:00", "pid": 100517, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.34, "name": "ffmpeg", "pid": 100517, "rss_mb": 139.34, "state": "S (sleeping)", "threads": 106}, "restart_count": 564, "restart_count_per_hour": 31, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 565}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 310.699, "sample_count": 1346, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1457623, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:22:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.585, "last_frame_pushed_at": "2026-07-01T11:22:11+09:00", "last_frame_read_age_sec": 4.764, "last_frame_read_at": "2026-07-01T11:22:11+09:00", "mjpeg": {"age_sec": 4.569, "exists": true, "mtime": "2026-07-01T11:22:11.656616+09:00", "mtime_epoch": 1782872531.656616, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176476}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:22:16+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5060.871, "name": "celery", "pid": 267, "rss_mb": 5060.871, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.301, "pushed": 4.301, "read": 4.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.553, "count": 129, "max_ms": 46.526, "p95_ms": 42.047}, "gstreamer_push": {"avg_ms": 0.881, "count": 129, "max_ms": 1.184, "p95_ms": 0.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 28.778, "count": 129, "max_ms": 38.277, "p95_ms": 34.835}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.466, "count": 129, "max_ms": 89.727, "p95_ms": 81.039}, "yolo11_640_fire_11class": {"avg_ms": 45.975, "count": 129, "max_ms": 67.834, "p95_ms": 60.65}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.053, "count": 129, "max_ms": 87.014, "p95_ms": 82.605}}, "truncated_count": 0}, "processing_total": {"avg_ms": 187.065, "count": 129, "max_ms": 230.257, "p95_ms": 216.959}, "read_interval": {"avg_ms": 197.06, "count": 129, "max_ms": 239.559, "p95_ms": 227.355}, "triton_wait": {"avg_ms": 62.82, "count": 129, "max_ms": 89.737, "p95_ms": 85.066}, "visualize": {"avg_ms": 6.225, "count": 129, "max_ms": 8.671, "p95_ms": 7.212}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 78, "tracking_summary": 13925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10896.09, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557a26746680] error while decoding MB 53 44, bytestream -21 [in#0/rtsp @ 0x557a2657ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557a2657ee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 217, "h264_decode_error_count": 224, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:23:37+09:00", "last_error_summary": "[h264 @ 0x557a26746680] error while decoding MB 53 44, bytestream -21 [in#0/rtsp @ 0x557a2657ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557a2657ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:15:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 486.269, "last_started_at": "2026-07-01T11:15:31+09:00", "pid": 99984, "process": {"exists": false, "pid": 99984}, "restart_count": 216, "restart_count_per_hour": 5, "rtsp_timeout_count": 358, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 217}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.652, "sample_count": 2632, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1671096, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:23:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.086, "last_frame_pushed_at": "2026-07-01T11:23:37+09:00", "last_frame_read_age_sec": 0.073, "last_frame_read_at": "2026-07-01T11:23:37+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-01T11:23:37.417979+09:00", "mtime_epoch": 1782872617.4179788, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238201}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:23:37+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1852.609, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.709, "pushed": 0.709, "read": 0.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.322, "count": 12, "max_ms": 21.309, "p95_ms": 21.309}, "gstreamer_push": {"avg_ms": 1.206, "count": 12, "max_ms": 1.337, "p95_ms": 1.337}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.166, "count": 12, "max_ms": 53.896, "p95_ms": 53.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.923, "count": 12, "max_ms": 88.08, "p95_ms": 88.08}, "yolo11_640_fire_11class": {"avg_ms": 50.544, "count": 12, "max_ms": 63.424, "p95_ms": 63.424}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.639, "count": 12, "max_ms": 85.354, "p95_ms": 85.354}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.073, "count": 12, "max_ms": 116.526, "p95_ms": 116.526}, "read_interval": {"avg_ms": 2615.981, "count": 13, "max_ms": 17151.537, "p95_ms": 17151.537}, "triton_wait": {"avg_ms": 59.934, "count": 12, "max_ms": 88.091, "p95_ms": 88.091}, "visualize": {"avg_ms": 3.32, "count": 12, "max_ms": 3.455, "p95_ms": 3.455}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.929, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 78, "tracking_summary": 13925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10895.863, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:23:37+09:00", "last_error_summary": "[h264 @ 0x557a26746680] error while decoding MB 53 44, bytestream -21 [in#0/rtsp @ 0x557a2657ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557a2657ee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 217, "h264_decode_error_count": 224, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:23:37+09:00", "last_error_summary": "[h264 @ 0x557a26746680] error while decoding MB 53 44, bytestream -21 [in#0/rtsp @ 0x557a2657ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557a2657ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:23:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 486.291, "last_started_at": "2026-07-01T11:15:31+09:00", "pid": 99984, "process": {"exists": false, "pid": 99984}, "restart_count": 216, "restart_count_per_hour": 5, "rtsp_timeout_count": 358, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 217}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.652, "sample_count": 2633, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1671097, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:23:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T11:23:37+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T11:23:37+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T11:23:37.417979+09:00", "mtime_epoch": 1782872617.4179788, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238201}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:23:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1852.609, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.767, "pushed": 0.767, "read": 0.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.572, "count": 13, "max_ms": 23.575, "p95_ms": 23.575}, "gstreamer_push": {"avg_ms": 1.21, "count": 13, "max_ms": 1.337, "p95_ms": 1.337}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.535, "count": 13, "max_ms": 53.896, "p95_ms": 53.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.431, "count": 13, "max_ms": 88.08, "p95_ms": 88.08}, "yolo11_640_fire_11class": {"avg_ms": 50.354, "count": 13, "max_ms": 63.424, "p95_ms": 63.424}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.161, "count": 13, "max_ms": 85.354, "p95_ms": 85.354}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.953, "count": 13, "max_ms": 116.526, "p95_ms": 116.526}, "read_interval": {"avg_ms": 2615.981, "count": 13, "max_ms": 17151.537, "p95_ms": 17151.537}, "triton_wait": {"avg_ms": 59.443, "count": 13, "max_ms": 88.091, "p95_ms": 88.091}, "visualize": {"avg_ms": 3.418, "count": 13, "max_ms": 4.592, "p95_ms": 4.592}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.951, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 78, "tracking_summary": 13925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10895.848, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 217, "h264_decode_error_count": 224, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:23:37+09:00", "last_error_summary": "[h264 @ 0x557a26746680] error while decoding MB 53 44, bytestream -21 [in#0/rtsp @ 0x557a2657ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557a2657ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:23:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 486.298, "last_started_at": "2026-07-01T11:15:31+09:00", "pid": 99984, "process": {"exists": false, "pid": 99984}, "restart_count": 216, "restart_count_per_hour": 5, "rtsp_timeout_count": 358, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 217}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.652, "sample_count": 2633, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1671097, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:23:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T11:23:37+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T11:23:37+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-01T11:23:37.417979+09:00", "mtime_epoch": 1782872617.4179788, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238201}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:23:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1852.609, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.767, "pushed": 0.767, "read": 0.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.572, "count": 13, "max_ms": 23.575, "p95_ms": 23.575}, "gstreamer_push": {"avg_ms": 1.21, "count": 13, "max_ms": 1.337, "p95_ms": 1.337}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.535, "count": 13, "max_ms": 53.896, "p95_ms": 53.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.431, "count": 13, "max_ms": 88.08, "p95_ms": 88.08}, "yolo11_640_fire_11class": {"avg_ms": 50.354, "count": 13, "max_ms": 63.424, "p95_ms": 63.424}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.161, "count": 13, "max_ms": 85.354, "p95_ms": 85.354}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.953, "count": 13, "max_ms": 116.526, "p95_ms": 116.526}, "read_interval": {"avg_ms": 2615.981, "count": 13, "max_ms": 17151.537, "p95_ms": 17151.537}, "triton_wait": {"avg_ms": 59.443, "count": 13, "max_ms": 88.091, "p95_ms": 88.091}, "visualize": {"avg_ms": 3.418, "count": 13, "max_ms": 4.592, "p95_ms": 4.592}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.958, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 78, "tracking_summary": 13925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 11019.254, "limit_mb": 8796093022207.996}, "detail": {"pid": 100624, "restart_count": 217, "start_count": 218}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 217, "h264_decode_error_count": 224, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:23:37+09:00", "last_error_summary": "[h264 @ 0x557a26746680] error while decoding MB 53 44, bytestream -21 [in#0/rtsp @ 0x557a2657ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557a2657ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:23:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:23:40+09:00", "pid": 100624, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.824, "name": "ffmpeg", "pid": 100624, "rss_mb": 156.66, "state": "R (running)", "threads": 49}, "restart_count": 217, "restart_count_per_hour": 6, "rtsp_timeout_count": 358, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 218}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.652, "sample_count": 2585, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1671097, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:23:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.929, "last_frame_pushed_at": "2026-07-01T11:23:37+09:00", "last_frame_read_age_sec": 5.018, "last_frame_read_at": "2026-07-01T11:23:37+09:00", "mjpeg": {"age_sec": 4.911, "exists": true, "mtime": "2026-07-01T11:23:37.521981+09:00", "mtime_epoch": 1782872617.5219805, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238073}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:23:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1852.609, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.594, "pushed": 0.594, "read": 0.594}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.572, "count": 13, "max_ms": 23.575, "p95_ms": 23.575}, "gstreamer_push": {"avg_ms": 1.21, "count": 13, "max_ms": 1.337, "p95_ms": 1.337}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.535, "count": 13, "max_ms": 53.896, "p95_ms": 53.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.431, "count": 13, "max_ms": 88.08, "p95_ms": 88.08}, "yolo11_640_fire_11class": {"avg_ms": 50.354, "count": 13, "max_ms": 63.424, "p95_ms": 63.424}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.161, "count": 13, "max_ms": 85.354, "p95_ms": 85.354}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.953, "count": 13, "max_ms": 116.526, "p95_ms": 116.526}, "read_interval": {"avg_ms": 2615.981, "count": 13, "max_ms": 17151.537, "p95_ms": 17151.537}, "triton_wait": {"avg_ms": 59.443, "count": 13, "max_ms": 88.091, "p95_ms": 88.091}, "visualize": {"avg_ms": 3.418, "count": 13, "max_ms": 4.592, "p95_ms": 4.592}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.874, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12152}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10960.281, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5604906224c0] error while decoding MB 2 20, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 515, "h264_decode_error_count": 1469, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:24:17+09:00", "last_error_summary": "[h264 @ 0x5604906224c0] error while decoding MB 2 20, bytestream -7", "last_exit_at": "2026-07-01T11:22:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.313, "last_started_at": "2026-07-01T11:22:14+09:00", "pid": 100517, "process": {"exists": false, "pid": 100517}, "restart_count": 564, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 565}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.533, "sample_count": 1483, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1458263, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:24:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.043, "last_frame_pushed_at": "2026-07-01T11:24:17+09:00", "last_frame_read_age_sec": 0.033, "last_frame_read_at": "2026-07-01T11:24:17+09:00", "mjpeg": {"age_sec": 0.026, "exists": true, "mtime": "2026-07-01T11:24:17.510618+09:00", "mtime_epoch": 1782872657.510618, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173671}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:24:17+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5061.129, "name": "celery", "pid": 267, "rss_mb": 5061.129, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 160, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.341, "pushed": 5.341, "read": 5.341}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.759, "count": 159, "max_ms": 42.301, "p95_ms": 38.959}, "gstreamer_push": {"avg_ms": 0.897, "count": 160, "max_ms": 1.062, "p95_ms": 1.004}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.792, "count": 159, "max_ms": 42.324, "p95_ms": 36.839}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.945, "count": 159, "max_ms": 88.353, "p95_ms": 76.782}, "yolo11_640_fire_11class": {"avg_ms": 45.393, "count": 159, "max_ms": 63.426, "p95_ms": 54.759}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.437, "count": 159, "max_ms": 82.481, "p95_ms": 72.383}}, "truncated_count": 0}, "processing_total": {"avg_ms": 178.441, "count": 160, "max_ms": 233.046, "p95_ms": 199.691}, "read_interval": {"avg_ms": 188.476, "count": 160, "max_ms": 242.632, "p95_ms": 210.201}, "triton_wait": {"avg_ms": 61.031, "count": 159, "max_ms": 88.366, "p95_ms": 77.05}, "visualize": {"avg_ms": 6.058, "count": 159, "max_ms": 8.276, "p95_ms": 7.071}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12152}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10959.605, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:24:17+09:00", "last_error_summary": "[h264 @ 0x5604906224c0] error while decoding MB 2 20, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 515, "h264_decode_error_count": 1469, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:24:17+09:00", "last_error_summary": "[h264 @ 0x5604906224c0] error while decoding MB 2 20, bytestream -7", "last_exit_at": "2026-07-01T11:24:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.447, "last_started_at": "2026-07-01T11:22:14+09:00", "pid": 100517, "process": {"exists": false, "pid": 100517}, "restart_count": 564, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 565}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.42, "sample_count": 1483, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1458264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:24:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T11:24:17+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T11:24:17+09:00", "mjpeg": {"age_sec": 0.16, "exists": true, "mtime": "2026-07-01T11:24:17.510618+09:00", "mtime_epoch": 1782872657.510618, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173671}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:24:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5061.129, "name": "celery", "pid": 267, "rss_mb": 5061.129, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 159, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.355, "pushed": 5.355, "read": 5.322}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.764, "count": 160, "max_ms": 42.301, "p95_ms": 38.909}, "gstreamer_push": {"avg_ms": 0.898, "count": 160, "max_ms": 1.062, "p95_ms": 1.004}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.797, "count": 160, "max_ms": 42.324, "p95_ms": 36.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.96, "count": 160, "max_ms": 88.353, "p95_ms": 74.896}, "yolo11_640_fire_11class": {"avg_ms": 45.432, "count": 160, "max_ms": 63.426, "p95_ms": 53.847}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.436, "count": 160, "max_ms": 82.481, "p95_ms": 71.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 177.993, "count": 160, "max_ms": 227.924, "p95_ms": 198.725}, "read_interval": {"avg_ms": 188.135, "count": 159, "max_ms": 238.649, "p95_ms": 210.201}, "triton_wait": {"avg_ms": 61.046, "count": 160, "max_ms": 88.366, "p95_ms": 76.794}, "visualize": {"avg_ms": 6.055, "count": 160, "max_ms": 8.276, "p95_ms": 6.975}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12152}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10959.648, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 515, "h264_decode_error_count": 1469, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:24:17+09:00", "last_error_summary": "[h264 @ 0x5604906224c0] error while decoding MB 2 20, bytestream -7", "last_exit_at": "2026-07-01T11:24:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.45, "last_started_at": "2026-07-01T11:22:14+09:00", "pid": 100517, "process": {"exists": false, "pid": 100517}, "restart_count": 564, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 565}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.42, "sample_count": 1483, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1458264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:24:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T11:24:17+09:00", "last_frame_read_age_sec": 0.171, "last_frame_read_at": "2026-07-01T11:24:17+09:00", "mjpeg": {"age_sec": 0.164, "exists": true, "mtime": "2026-07-01T11:24:17.510618+09:00", "mtime_epoch": 1782872657.510618, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173671}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:24:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5061.129, "name": "celery", "pid": 267, "rss_mb": 5061.129, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 159, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.354, "pushed": 5.354, "read": 5.321}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.764, "count": 160, "max_ms": 42.301, "p95_ms": 38.909}, "gstreamer_push": {"avg_ms": 0.898, "count": 160, "max_ms": 1.062, "p95_ms": 1.004}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.797, "count": 160, "max_ms": 42.324, "p95_ms": 36.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.96, "count": 160, "max_ms": 88.353, "p95_ms": 74.896}, "yolo11_640_fire_11class": {"avg_ms": 45.432, "count": 160, "max_ms": 63.426, "p95_ms": 53.847}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.436, "count": 160, "max_ms": 82.481, "p95_ms": 71.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 177.993, "count": 160, "max_ms": 227.924, "p95_ms": 198.725}, "read_interval": {"avg_ms": 188.135, "count": 159, "max_ms": 238.649, "p95_ms": 210.201}, "triton_wait": {"avg_ms": 61.046, "count": 160, "max_ms": 88.366, "p95_ms": 76.794}, "visualize": {"avg_ms": 6.055, "count": 160, "max_ms": 8.276, "p95_ms": 6.975}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12152}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11065.094, "limit_mb": 8796093022207.996}, "detail": {"pid": 100776, "restart_count": 565, "start_count": 566}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 515, "h264_decode_error_count": 1469, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:24:17+09:00", "last_error_summary": "[h264 @ 0x5604906224c0] error while decoding MB 2 20, bytestream -7", "last_exit_at": "2026-07-01T11:24:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:24:20+09:00", "pid": 100776, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.82, "name": "ffmpeg", "pid": 100776, "rss_mb": 136.82, "state": "S (sleeping)", "threads": 106}, "restart_count": 565, "restart_count_per_hour": 31, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 566}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.01, "sample_count": 1460, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1458264, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:24:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.967, "last_frame_pushed_at": "2026-07-01T11:24:17+09:00", "last_frame_read_age_sec": 5.13, "last_frame_read_at": "2026-07-01T11:24:17+09:00", "mjpeg": {"age_sec": 4.95, "exists": true, "mtime": "2026-07-01T11:24:17.682621+09:00", "mtime_epoch": 1782872657.6826208, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171837}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:24:22+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5061.129, "name": "celery", "pid": 267, "rss_mb": 5061.129, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.491, "pushed": 4.491, "read": 4.458}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.77, "count": 134, "max_ms": 42.301, "p95_ms": 39.318}, "gstreamer_push": {"avg_ms": 0.899, "count": 134, "max_ms": 1.062, "p95_ms": 1.019}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.908, "count": 134, "max_ms": 42.324, "p95_ms": 37.008}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.832, "count": 134, "max_ms": 88.353, "p95_ms": 74.896}, "yolo11_640_fire_11class": {"avg_ms": 45.24, "count": 134, "max_ms": 63.426, "p95_ms": 54.759}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.454, "count": 134, "max_ms": 82.481, "p95_ms": 72.383}}, "truncated_count": 0}, "processing_total": {"avg_ms": 176.958, "count": 134, "max_ms": 211.272, "p95_ms": 196.466}, "read_interval": {"avg_ms": 187.097, "count": 133, "max_ms": 222.047, "p95_ms": 206.27}, "triton_wait": {"avg_ms": 60.932, "count": 134, "max_ms": 88.366, "p95_ms": 76.794}, "visualize": {"avg_ms": 6.02, "count": 134, "max_ms": 8.276, "p95_ms": 6.915}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.834, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12157}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10961.285, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556f753c9a00] error while decoding MB 69 22, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 516, "h264_decode_error_count": 1470, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:26:26+09:00", "last_error_summary": "[h264 @ 0x556f753c9a00] error while decoding MB 69 22, bytestream -10", "last_exit_at": "2026-07-01T11:24:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.489, "last_started_at": "2026-07-01T11:24:20+09:00", "pid": 100776, "process": {"exists": false, "pid": 100776}, "restart_count": 565, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 566}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 205.681, "sample_count": 1526, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1458918, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:26:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.094, "last_frame_pushed_at": "2026-07-01T11:26:26+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-01T11:26:26+09:00", "mjpeg": {"age_sec": 0.076, "exists": true, "mtime": "2026-07-01T11:26:26.044674+09:00", "mtime_epoch": 1782872786.0446744, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174671}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:26:26+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5061.387, "name": "celery", "pid": 267, "rss_mb": 5061.387, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 158, "pushed_frames": 158, "read_frames": 158, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.274, "pushed": 5.274, "read": 5.274}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.563, "count": 157, "max_ms": 46.39, "p95_ms": 41.51}, "gstreamer_push": {"avg_ms": 0.894, "count": 158, "max_ms": 1.289, "p95_ms": 1.0}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.443, "count": 157, "max_ms": 37.983, "p95_ms": 34.319}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.656, "count": 157, "max_ms": 89.941, "p95_ms": 79.16}, "yolo11_640_fire_11class": {"avg_ms": 47.465, "count": 157, "max_ms": 68.761, "p95_ms": 57.532}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.934, "count": 157, "max_ms": 87.119, "p95_ms": 76.955}}, "truncated_count": 0}, "processing_total": {"avg_ms": 180.295, "count": 158, "max_ms": 222.197, "p95_ms": 208.539}, "read_interval": {"avg_ms": 190.182, "count": 158, "max_ms": 231.494, "p95_ms": 216.435}, "triton_wait": {"avg_ms": 63.047, "count": 157, "max_ms": 89.952, "p95_ms": 79.404}, "visualize": {"avg_ms": 6.156, "count": 157, "max_ms": 8.843, "p95_ms": 7.764}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12157}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10960.785, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:26:26+09:00", "last_error_summary": "[h264 @ 0x556f753c9a00] error while decoding MB 69 22, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 516, "h264_decode_error_count": 1470, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:26:26+09:00", "last_error_summary": "[h264 @ 0x556f753c9a00] error while decoding MB 69 22, bytestream -10", "last_exit_at": "2026-07-01T11:26:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.601, "last_started_at": "2026-07-01T11:24:20+09:00", "pid": 100776, "process": {"exists": false, "pid": 100776}, "restart_count": 565, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 566}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 205.681, "sample_count": 1527, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1458919, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:26:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:26:26+09:00", "last_frame_read_age_sec": 0.198, "last_frame_read_at": "2026-07-01T11:26:26+09:00", "mjpeg": {"age_sec": 0.188, "exists": true, "mtime": "2026-07-01T11:26:26.044674+09:00", "mtime_epoch": 1782872786.0446744, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174671}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:26:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5061.387, "name": "celery", "pid": 267, "rss_mb": 5061.387, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 158, "pushed_frames": 158, "read_frames": 157, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.286, "pushed": 5.286, "read": 5.253}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.56, "count": 158, "max_ms": 46.39, "p95_ms": 41.51}, "gstreamer_push": {"avg_ms": 0.893, "count": 158, "max_ms": 1.289, "p95_ms": 0.996}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.429, "count": 158, "max_ms": 37.983, "p95_ms": 34.319}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.628, "count": 158, "max_ms": 89.941, "p95_ms": 79.16}, "yolo11_640_fire_11class": {"avg_ms": 47.444, "count": 158, "max_ms": 68.761, "p95_ms": 57.532}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.905, "count": 158, "max_ms": 87.119, "p95_ms": 76.955}}, "truncated_count": 0}, "processing_total": {"avg_ms": 180.419, "count": 158, "max_ms": 222.197, "p95_ms": 208.539}, "read_interval": {"avg_ms": 190.212, "count": 157, "max_ms": 231.494, "p95_ms": 216.435}, "triton_wait": {"avg_ms": 63.017, "count": 158, "max_ms": 89.952, "p95_ms": 79.404}, "visualize": {"avg_ms": 6.19, "count": 158, "max_ms": 11.509, "p95_ms": 7.893}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12157}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 10960.902, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 516, "h264_decode_error_count": 1470, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:26:26+09:00", "last_error_summary": "[h264 @ 0x556f753c9a00] error while decoding MB 69 22, bytestream -10", "last_exit_at": "2026-07-01T11:26:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.605, "last_started_at": "2026-07-01T11:24:20+09:00", "pid": 100776, "process": {"exists": false, "pid": 100776}, "restart_count": 565, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 566}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 205.681, "sample_count": 1527, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1458919, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:26:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T11:26:26+09:00", "last_frame_read_age_sec": 0.202, "last_frame_read_at": "2026-07-01T11:26:26+09:00", "mjpeg": {"age_sec": 0.192, "exists": true, "mtime": "2026-07-01T11:26:26.044674+09:00", "mtime_epoch": 1782872786.0446744, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174671}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:26:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5061.387, "name": "celery", "pid": 267, "rss_mb": 5061.387, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 158, "pushed_frames": 158, "read_frames": 157, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.286, "pushed": 5.286, "read": 5.252}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.56, "count": 158, "max_ms": 46.39, "p95_ms": 41.51}, "gstreamer_push": {"avg_ms": 0.893, "count": 158, "max_ms": 1.289, "p95_ms": 0.996}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.429, "count": 158, "max_ms": 37.983, "p95_ms": 34.319}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.628, "count": 158, "max_ms": 89.941, "p95_ms": 79.16}, "yolo11_640_fire_11class": {"avg_ms": 47.444, "count": 158, "max_ms": 68.761, "p95_ms": 57.532}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.905, "count": 158, "max_ms": 87.119, "p95_ms": 76.955}}, "truncated_count": 0}, "processing_total": {"avg_ms": 180.419, "count": 158, "max_ms": 222.197, "p95_ms": 208.539}, "read_interval": {"avg_ms": 190.212, "count": 157, "max_ms": 231.494, "p95_ms": 216.435}, "triton_wait": {"avg_ms": 63.017, "count": 158, "max_ms": 89.952, "p95_ms": 79.404}, "visualize": {"avg_ms": 6.19, "count": 158, "max_ms": 11.509, "p95_ms": 7.893}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12157}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11068.52, "limit_mb": 8796093022207.996}, "detail": {"pid": 100883, "restart_count": 566, "start_count": 567}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 516, "h264_decode_error_count": 1470, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:26:26+09:00", "last_error_summary": "[h264 @ 0x556f753c9a00] error while decoding MB 69 22, bytestream -10", "last_exit_at": "2026-07-01T11:26:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:26:29+09:00", "pid": 100883, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.828, "name": "ffmpeg", "pid": 100883, "rss_mb": 138.828, "state": "S (sleeping)", "threads": 106}, "restart_count": 566, "restart_count_per_hour": 31, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 567}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 205.681, "sample_count": 1500, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1458919, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:26:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.957, "last_frame_pushed_at": "2026-07-01T11:26:26+09:00", "last_frame_read_age_sec": 5.151, "last_frame_read_at": "2026-07-01T11:26:26+09:00", "mjpeg": {"age_sec": 4.942, "exists": true, "mtime": "2026-07-01T11:26:26.244678+09:00", "mtime_epoch": 1782872786.2446778, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174679}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:26:31+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5061.387, "name": "celery", "pid": 267, "rss_mb": 5061.387, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.401, "pushed": 4.401, "read": 4.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.102, "count": 131, "max_ms": 46.39, "p95_ms": 40.315}, "gstreamer_push": {"avg_ms": 0.891, "count": 132, "max_ms": 1.289, "p95_ms": 0.995}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.335, "count": 131, "max_ms": 37.983, "p95_ms": 34.319}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.339, "count": 131, "max_ms": 89.941, "p95_ms": 79.392}, "yolo11_640_fire_11class": {"avg_ms": 47.599, "count": 131, "max_ms": 68.761, "p95_ms": 61.18}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.349, "count": 131, "max_ms": 87.119, "p95_ms": 83.726}}, "truncated_count": 0}, "processing_total": {"avg_ms": 181.245, "count": 132, "max_ms": 222.197, "p95_ms": 213.932}, "read_interval": {"avg_ms": 190.944, "count": 131, "max_ms": 231.494, "p95_ms": 223.032}, "triton_wait": {"avg_ms": 63.675, "count": 131, "max_ms": 89.952, "p95_ms": 83.737}, "visualize": {"avg_ms": 6.094, "count": 131, "max_ms": 11.509, "p95_ms": 7.167}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_112715_4e5a5230.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 11306.488, "limit_mb": 8796093022207.996}, "detail": {"pid": 100992, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:27:15+09:00", "pid": 100992, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.047, "name": "ffmpeg", "pid": 100992, "rss_mb": 137.047, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.938}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "4e5a5230", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": null, "exists": false, "mtime": null, "mtime_epoch": null, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": null}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T11:27:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1122.781, "name": "celery", "pid": 239, "rss_mb": 1087.367, "state": "R (running)", "threads": 120}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:27:15+09:00", "state": "running", "task_id": "145b0ffe-308e-4f92-9b1f-b1f314df881c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11046.727, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558568393200] error while decoding MB 65 23, bytestream -24", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 517, "h264_decode_error_count": 1471, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:28:28+09:00", "last_error_summary": "[h264 @ 0x558568393200] error while decoding MB 65 23, bytestream -24", "last_exit_at": "2026-07-01T11:26:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.661, "last_started_at": "2026-07-01T11:26:29+09:00", "pid": 100883, "process": {"exists": false, "pid": 100883}, "restart_count": 566, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 567}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.586, "sample_count": 1502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1459508, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:28:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T11:28:28+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T11:28:28+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T11:28:28.802648+09:00", "mtime_epoch": 1782872908.8026476, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175474}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:28:28+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5061.902, "name": "celery", "pid": 267, "rss_mb": 5061.902, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 156, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.22, "pushed": 5.22, "read": 5.22}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.556, "count": 156, "max_ms": 54.763, "p95_ms": 39.774}, "gstreamer_push": {"avg_ms": 0.891, "count": 156, "max_ms": 1.45, "p95_ms": 0.996}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.373, "count": 156, "max_ms": 36.992, "p95_ms": 33.715}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.169, "count": 156, "max_ms": 89.372, "p95_ms": 81.456}, "yolo11_640_fire_11class": {"avg_ms": 45.495, "count": 156, "max_ms": 65.429, "p95_ms": 54.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.298, "count": 156, "max_ms": 86.94, "p95_ms": 80.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 182.371, "count": 156, "max_ms": 222.8, "p95_ms": 204.582}, "read_interval": {"avg_ms": 192.356, "count": 156, "max_ms": 232.225, "p95_ms": 214.562}, "triton_wait": {"avg_ms": 62.288, "count": 156, "max_ms": 89.385, "p95_ms": 84.234}, "visualize": {"avg_ms": 6.11, "count": 156, "max_ms": 8.158, "p95_ms": 7.088}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.886, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11046.176, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:28:28+09:00", "last_error_summary": "[h264 @ 0x558568393200] error while decoding MB 65 23, bytestream -24", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 517, "h264_decode_error_count": 1471, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:28:28+09:00", "last_error_summary": "[h264 @ 0x558568393200] error while decoding MB 65 23, bytestream -24", "last_exit_at": "2026-07-01T11:28:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.808, "last_started_at": "2026-07-01T11:26:29+09:00", "pid": 100883, "process": {"exists": false, "pid": 100883}, "restart_count": 566, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 567}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.586, "sample_count": 1502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1459509, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:28:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:28:28+09:00", "last_frame_read_age_sec": 0.199, "last_frame_read_at": "2026-07-01T11:28:28+09:00", "mjpeg": {"age_sec": 0.191, "exists": true, "mtime": "2026-07-01T11:28:28.802648+09:00", "mtime_epoch": 1782872908.8026476, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175474}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:28:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5061.902, "name": "celery", "pid": 267, "rss_mb": 5061.902, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.223, "pushed": 5.223, "read": 5.19}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.564, "count": 156, "max_ms": 54.763, "p95_ms": 39.774}, "gstreamer_push": {"avg_ms": 0.89, "count": 156, "max_ms": 1.45, "p95_ms": 0.996}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.415, "count": 156, "max_ms": 36.992, "p95_ms": 33.972}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.184, "count": 156, "max_ms": 89.372, "p95_ms": 81.456}, "yolo11_640_fire_11class": {"avg_ms": 45.536, "count": 156, "max_ms": 65.429, "p95_ms": 55.868}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.315, "count": 156, "max_ms": 86.94, "p95_ms": 80.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 182.505, "count": 156, "max_ms": 222.8, "p95_ms": 204.582}, "read_interval": {"avg_ms": 192.415, "count": 155, "max_ms": 232.225, "p95_ms": 214.562}, "triton_wait": {"avg_ms": 62.302, "count": 156, "max_ms": 89.385, "p95_ms": 84.234}, "visualize": {"avg_ms": 6.113, "count": 156, "max_ms": 8.158, "p95_ms": 7.088}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11046.168, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 517, "h264_decode_error_count": 1471, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:28:28+09:00", "last_error_summary": "[h264 @ 0x558568393200] error while decoding MB 65 23, bytestream -24", "last_exit_at": "2026-07-01T11:28:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.812, "last_started_at": "2026-07-01T11:26:29+09:00", "pid": 100883, "process": {"exists": false, "pid": 100883}, "restart_count": 566, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 567}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.586, "sample_count": 1502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1459509, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:28:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T11:28:28+09:00", "last_frame_read_age_sec": 0.202, "last_frame_read_at": "2026-07-01T11:28:28+09:00", "mjpeg": {"age_sec": 0.195, "exists": true, "mtime": "2026-07-01T11:28:28.802648+09:00", "mtime_epoch": 1782872908.8026476, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175474}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:28:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5061.902, "name": "celery", "pid": 267, "rss_mb": 5061.902, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.223, "pushed": 5.223, "read": 5.189}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.564, "count": 156, "max_ms": 54.763, "p95_ms": 39.774}, "gstreamer_push": {"avg_ms": 0.89, "count": 156, "max_ms": 1.45, "p95_ms": 0.996}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.415, "count": 156, "max_ms": 36.992, "p95_ms": 33.972}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.184, "count": 156, "max_ms": 89.372, "p95_ms": 81.456}, "yolo11_640_fire_11class": {"avg_ms": 45.536, "count": 156, "max_ms": 65.429, "p95_ms": 55.868}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.315, "count": 156, "max_ms": 86.94, "p95_ms": 80.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 182.505, "count": 156, "max_ms": 222.8, "p95_ms": 204.582}, "read_interval": {"avg_ms": 192.415, "count": 155, "max_ms": 232.225, "p95_ms": 214.562}, "triton_wait": {"avg_ms": 62.302, "count": 156, "max_ms": 89.385, "p95_ms": 84.234}, "visualize": {"avg_ms": 6.113, "count": 156, "max_ms": 8.158, "p95_ms": 7.088}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11151.254, "limit_mb": 8796093022207.996}, "detail": {"pid": 101145, "restart_count": 567, "start_count": 568}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 517, "h264_decode_error_count": 1471, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:28:28+09:00", "last_error_summary": "[h264 @ 0x558568393200] error while decoding MB 65 23, bytestream -24", "last_exit_at": "2026-07-01T11:28:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:28:31+09:00", "pid": 101145, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.875, "name": "ffmpeg", "pid": 101145, "rss_mb": 136.875, "state": "S (sleeping)", "threads": 106}, "restart_count": 567, "restart_count_per_hour": 31, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 568}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.601, "sample_count": 1476, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1459509, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:28:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.6, "last_frame_pushed_at": "2026-07-01T11:28:28+09:00", "last_frame_read_age_sec": 4.795, "last_frame_read_at": "2026-07-01T11:28:28+09:00", "mjpeg": {"age_sec": 4.584, "exists": true, "mtime": "2026-07-01T11:28:29.006651+09:00", "mtime_epoch": 1782872909.006651, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:28:33+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5061.902, "name": "celery", "pid": 267, "rss_mb": 5061.902, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.386, "pushed": 4.386, "read": 4.352}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.67, "count": 131, "max_ms": 54.763, "p95_ms": 39.677}, "gstreamer_push": {"avg_ms": 0.885, "count": 131, "max_ms": 1.45, "p95_ms": 0.965}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 29.68, "count": 131, "max_ms": 36.992, "p95_ms": 34.427}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.67, "count": 131, "max_ms": 89.372, "p95_ms": 81.879}, "yolo11_640_fire_11class": {"avg_ms": 46.636, "count": 131, "max_ms": 65.429, "p95_ms": 56.45}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.67, "count": 131, "max_ms": 85.687, "p95_ms": 80.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.296, "count": 131, "max_ms": 222.8, "p95_ms": 204.96}, "read_interval": {"avg_ms": 194.169, "count": 130, "max_ms": 232.225, "p95_ms": 214.835}, "triton_wait": {"avg_ms": 62.741, "count": 131, "max_ms": 89.385, "p95_ms": 84.234}, "visualize": {"avg_ms": 6.182, "count": 131, "max_ms": 8.158, "p95_ms": 7.09}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11053.773, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:28:28+09:00", "last_error_summary": "[h264 @ 0x558568393200] error while decoding MB 65 23, bytestream -24", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 517, "h264_decode_error_count": 1471, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:28:28+09:00", "last_error_summary": "[h264 @ 0x558568393200] error while decoding MB 65 23, bytestream -24", "last_exit_at": "2026-07-01T11:30:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.202, "last_started_at": "2026-07-01T11:28:31+09:00", "pid": 101145, "process": {"exists": false, "pid": 101145}, "restart_count": 567, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 568}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.164, "sample_count": 1475, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1460125, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:30:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:30:35+09:00", "last_frame_read_age_sec": 0.207, "last_frame_read_at": "2026-07-01T11:30:35+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T11:30:35.592747+09:00", "mtime_epoch": 1782873035.5927467, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177972}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:30:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5062.16, "name": "celery", "pid": 267, "rss_mb": 5062.16, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.993, "pushed": 4.993, "read": 4.959}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.728, "count": 149, "max_ms": 46.328, "p95_ms": 40.15}, "gstreamer_push": {"avg_ms": 0.887, "count": 149, "max_ms": 1.149, "p95_ms": 1.002}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.82, "count": 149, "max_ms": 54.598, "p95_ms": 38.302}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.686, "count": 149, "max_ms": 90.545, "p95_ms": 88.736}, "yolo11_640_fire_11class": {"avg_ms": 46.767, "count": 149, "max_ms": 66.798, "p95_ms": 63.501}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.382, "count": 149, "max_ms": 88.077, "p95_ms": 86.762}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.012, "count": 149, "max_ms": 267.62, "p95_ms": 239.588}, "read_interval": {"avg_ms": 201.525, "count": 148, "max_ms": 276.677, "p95_ms": 249.062}, "triton_wait": {"avg_ms": 64.993, "count": 149, "max_ms": 90.558, "p95_ms": 88.749}, "visualize": {"avg_ms": 6.418, "count": 149, "max_ms": 8.903, "p95_ms": 8.015}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.842, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11053.766, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 517, "h264_decode_error_count": 1471, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:28:28+09:00", "last_error_summary": "[h264 @ 0x558568393200] error while decoding MB 65 23, bytestream -24", "last_exit_at": "2026-07-01T11:30:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.206, "last_started_at": "2026-07-01T11:28:31+09:00", "pid": 101145, "process": {"exists": false, "pid": 101145}, "restart_count": 567, "restart_count_per_hour": 30, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 568}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.164, "sample_count": 1475, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1460125, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:30:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T11:30:35+09:00", "last_frame_read_age_sec": 0.211, "last_frame_read_at": "2026-07-01T11:30:35+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-01T11:30:35.592747+09:00", "mtime_epoch": 1782873035.5927467, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177972}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:30:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 5062.16, "name": "celery", "pid": 267, "rss_mb": 5062.16, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.992, "pushed": 4.992, "read": 4.959}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.728, "count": 149, "max_ms": 46.328, "p95_ms": 40.15}, "gstreamer_push": {"avg_ms": 0.887, "count": 149, "max_ms": 1.149, "p95_ms": 1.002}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.82, "count": 149, "max_ms": 54.598, "p95_ms": 38.302}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.686, "count": 149, "max_ms": 90.545, "p95_ms": 88.736}, "yolo11_640_fire_11class": {"avg_ms": 46.767, "count": 149, "max_ms": 66.798, "p95_ms": 63.501}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.382, "count": 149, "max_ms": 88.077, "p95_ms": 86.762}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.012, "count": 149, "max_ms": 267.62, "p95_ms": 239.588}, "read_interval": {"avg_ms": 201.525, "count": 148, "max_ms": 276.677, "p95_ms": 249.062}, "triton_wait": {"avg_ms": 64.993, "count": 149, "max_ms": 90.558, "p95_ms": 88.749}, "visualize": {"avg_ms": 6.418, "count": 149, "max_ms": 8.903, "p95_ms": 8.015}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260629_090810_27b97f08.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 443, "tracking_summary": 12167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 11163.77, "limit_mb": 8796093022207.996}, "detail": {"pid": 101312, "restart_count": 568, "start_count": 569}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 517, "h264_decode_error_count": 1471, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:28:28+09:00", "last_error_summary": "[h264 @ 0x558568393200] error while decoding MB 65 23, bytestream -24", "last_exit_at": "2026-07-01T11:30:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:30:38+09:00", "pid": 101312, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.617, "name": "ffmpeg", "pid": 101312, "rss_mb": 141.617, "state": "S (sleeping)", "threads": 106}, "restart_count": 568, "restart_count_per_hour": 31, "rtsp_timeout_count": 412, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 569}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.261, "sample_count": 1450, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1460125, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:30:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "27b97f08", "last_frame_pushed_age_sec": 4.471, "last_frame_pushed_at": "2026-07-01T11:30:35+09:00", "last_frame_read_age_sec": 4.675, "last_frame_read_at": "2026-07-01T11:30:35+09:00", "mjpeg": {"age_sec": 4.455, "exists": true, "mtime": "2026-07-01T11:30:35.804751+09:00", "mtime_epoch": 1782873035.8047507, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173189}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:30:40+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 5062.16, "name": "celery", "pid": 267, "rss_mb": 5062.16, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 3, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.167, "pushed": 4.2, "read": 4.167}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.85, "count": 125, "max_ms": 46.328, "p95_ms": 40.517}, "gstreamer_push": {"avg_ms": 0.885, "count": 126, "max_ms": 1.149, "p95_ms": 0.98}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.656, "count": 125, "max_ms": 43.183, "p95_ms": 38.302}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.273, "count": 125, "max_ms": 90.545, "p95_ms": 88.736}, "yolo11_640_fire_11class": {"avg_ms": 47.575, "count": 125, "max_ms": 66.798, "p95_ms": 64.048}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.199, "count": 125, "max_ms": 88.077, "p95_ms": 86.762}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.547, "count": 125, "max_ms": 267.62, "p95_ms": 244.507}, "read_interval": {"avg_ms": 203.877, "count": 125, "max_ms": 276.677, "p95_ms": 253.923}, "triton_wait": {"avg_ms": 65.616, "count": 125, "max_ms": 90.558, "p95_ms": 88.749}, "visualize": {"avg_ms": 6.478, "count": 125, "max_ms": 8.903, "p95_ms": 8.078}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-06-29T09:08:10+09:00", "state": "running", "task_id": "c8882907-09f2-4f9b-9fb8-402f8b2f62ca", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 79, "tracking_summary": 13961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10988.445, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55fd2a479200] missing picture in access unit with size 18 [h264 @ 0x55fd2ad7e800] no frame! [in#0/rtsp @ 0x55fd2a474e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fd2a474e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55fd2a479840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 218, "h264_decode_error_count": 225, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:31:33+09:00", "last_error_summary": "[NULL @ 0x55fd2a479200] missing picture in access unit with size 18 [h264 @ 0x55fd2ad7e800] no frame! [in#0/rtsp @ 0x55fd2a474e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fd2a474e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55fd2a479840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T11:23:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 472.822, "last_started_at": "2026-07-01T11:23:40+09:00", "pid": 100624, "process": {"exists": false, "pid": 100624}, "restart_count": 217, "restart_count_per_hour": 6, "rtsp_timeout_count": 360, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 218}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.747, "sample_count": 2555, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1675357, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:31:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T11:31:33+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T11:31:33+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T11:31:33.209853+09:00", "mtime_epoch": 1782873093.209853, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230080}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:31:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1855.258, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.455, "pushed": 0.455, "read": 0.49}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.866, "count": 13, "max_ms": 27.728, "p95_ms": 27.728}, "gstreamer_push": {"avg_ms": 1.178, "count": 13, "max_ms": 1.363, "p95_ms": 1.363}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.028, "count": 13, "max_ms": 53.595, "p95_ms": 53.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.008, "count": 13, "max_ms": 82.822, "p95_ms": 82.822}, "yolo11_640_fire_11class": {"avg_ms": 49.6, "count": 13, "max_ms": 62.515, "p95_ms": 62.515}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.986, "count": 13, "max_ms": 85.068, "p95_ms": 85.068}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.995, "count": 13, "max_ms": 113.907, "p95_ms": 113.907}, "read_interval": {"avg_ms": 2556.264, "count": 14, "max_ms": 17319.443, "p95_ms": 17319.443}, "triton_wait": {"avg_ms": 56.194, "count": 13, "max_ms": 85.08, "p95_ms": 85.08}, "visualize": {"avg_ms": 3.405, "count": 13, "max_ms": 3.868, "p95_ms": 3.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.561, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 79, "tracking_summary": 13961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10987.957, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:31:33+09:00", "last_error_summary": "[NULL @ 0x55fd2a479200] missing picture in access unit with size 18 [h264 @ 0x55fd2ad7e800] no frame! [in#0/rtsp @ 0x55fd2a474e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fd2a474e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55fd2a479840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 218, "h264_decode_error_count": 225, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:31:33+09:00", "last_error_summary": "[NULL @ 0x55fd2a479200] missing picture in access unit with size 18 [h264 @ 0x55fd2ad7e800] no frame! [in#0/rtsp @ 0x55fd2a474e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fd2a474e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55fd2a479840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T11:31:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 472.894, "last_started_at": "2026-07-01T11:23:40+09:00", "pid": 100624, "process": {"exists": false, "pid": 100624}, "restart_count": 217, "restart_count_per_hour": 6, "rtsp_timeout_count": 360, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 218}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.747, "sample_count": 2555, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1675358, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:31:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T11:31:33+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-01T11:31:33+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-01T11:31:33.209853+09:00", "mtime_epoch": 1782873093.209853, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230080}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:31:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1855.258, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.489, "pushed": 0.489, "read": 0.489}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.785, "count": 14, "max_ms": 27.728, "p95_ms": 27.728}, "gstreamer_push": {"avg_ms": 1.182, "count": 14, "max_ms": 1.363, "p95_ms": 1.363}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.799, "count": 14, "max_ms": 53.595, "p95_ms": 53.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.984, "count": 14, "max_ms": 83.671, "p95_ms": 83.671}, "yolo11_640_fire_11class": {"avg_ms": 50.222, "count": 14, "max_ms": 62.515, "p95_ms": 62.515}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.904, "count": 14, "max_ms": 85.068, "p95_ms": 85.068}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.864, "count": 14, "max_ms": 113.907, "p95_ms": 113.907}, "read_interval": {"avg_ms": 2556.264, "count": 14, "max_ms": 17319.443, "p95_ms": 17319.443}, "triton_wait": {"avg_ms": 58.157, "count": 14, "max_ms": 85.08, "p95_ms": 85.08}, "visualize": {"avg_ms": 3.395, "count": 14, "max_ms": 3.868, "p95_ms": 3.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.632, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 79, "tracking_summary": 13961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 10987.949, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 218, "h264_decode_error_count": 225, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:31:33+09:00", "last_error_summary": "[NULL @ 0x55fd2a479200] missing picture in access unit with size 18 [h264 @ 0x55fd2ad7e800] no frame! [in#0/rtsp @ 0x55fd2a474e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fd2a474e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55fd2a479840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T11:31:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 472.9, "last_started_at": "2026-07-01T11:23:40+09:00", "pid": 100624, "process": {"exists": false, "pid": 100624}, "restart_count": 217, "restart_count_per_hour": 6, "rtsp_timeout_count": 360, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 218}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.747, "sample_count": 2555, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1675358, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:31:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T11:31:33+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-01T11:31:33+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-01T11:31:33.209853+09:00", "mtime_epoch": 1782873093.209853, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230080}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:31:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1855.258, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.489, "pushed": 0.489, "read": 0.489}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.785, "count": 14, "max_ms": 27.728, "p95_ms": 27.728}, "gstreamer_push": {"avg_ms": 1.182, "count": 14, "max_ms": 1.363, "p95_ms": 1.363}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.799, "count": 14, "max_ms": 53.595, "p95_ms": 53.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.984, "count": 14, "max_ms": 83.671, "p95_ms": 83.671}, "yolo11_640_fire_11class": {"avg_ms": 50.222, "count": 14, "max_ms": 62.515, "p95_ms": 62.515}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.904, "count": 14, "max_ms": 85.068, "p95_ms": 85.068}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.864, "count": 14, "max_ms": 113.907, "p95_ms": 113.907}, "read_interval": {"avg_ms": 2556.264, "count": 14, "max_ms": 17319.443, "p95_ms": 17319.443}, "triton_wait": {"avg_ms": 58.157, "count": 14, "max_ms": 85.08, "p95_ms": 85.08}, "visualize": {"avg_ms": 3.395, "count": 14, "max_ms": 3.868, "p95_ms": 3.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.638, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260629_090809_b2785c18.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 79, "tracking_summary": 13961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 11009.074, "limit_mb": 8796093022207.996}, "detail": {"pid": 101419, "restart_count": 218, "start_count": 219}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 218, "h264_decode_error_count": 225, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:31:33+09:00", "last_error_summary": "[NULL @ 0x55fd2a479200] missing picture in access unit with size 18 [h264 @ 0x55fd2ad7e800] no frame! [in#0/rtsp @ 0x55fd2a474e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fd2a474e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55fd2a479840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T11:31:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:31:36+09:00", "pid": 101419, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.492, "name": "ffmpeg", "pid": 101419, "rss_mb": 58.492, "state": "S (sleeping)", "threads": 1}, "restart_count": 218, "restart_count_per_hour": 7, "rtsp_timeout_count": 360, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 219}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.348, "sample_count": 2513, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1675358, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:31:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "b2785c18", "last_frame_pushed_age_sec": 4.799, "last_frame_pushed_at": "2026-07-01T11:31:33+09:00", "last_frame_read_age_sec": 4.912, "last_frame_read_at": "2026-07-01T11:31:33+09:00", "mjpeg": {"age_sec": 4.782, "exists": true, "mtime": "2026-07-01T11:31:33.333855+09:00", "mtime_epoch": 1782873093.3338554, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 229733}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:31:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2138.227, "name": "celery", "pid": 237, "rss_mb": 1855.258, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.558, "pushed": 0.558, "read": 0.558}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 20.25, "count": 13, "max_ms": 21.761, "p95_ms": 21.761}, "gstreamer_push": {"avg_ms": 1.185, "count": 13, "max_ms": 1.363, "p95_ms": 1.363}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.975, "count": 13, "max_ms": 53.595, "p95_ms": 53.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.977, "count": 13, "max_ms": 83.671, "p95_ms": 83.671}, "yolo11_640_fire_11class": {"avg_ms": 50.156, "count": 13, "max_ms": 62.515, "p95_ms": 62.515}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.139, "count": 13, "max_ms": 85.068, "p95_ms": 85.068}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.293, "count": 13, "max_ms": 113.907, "p95_ms": 113.907}, "read_interval": {"avg_ms": 2193.137, "count": 13, "max_ms": 17319.443, "p95_ms": 17319.443}, "triton_wait": {"avg_ms": 58.164, "count": 13, "max_ms": 85.08, "p95_ms": 85.08}, "visualize": {"avg_ms": 3.358, "count": 13, "max_ms": 3.512, "p95_ms": 3.512}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.305, "window_s": 30.0}, "started_at": "2026-06-29T09:08:09+09:00", "state": "running", "task_id": "3bd3c942-be81-4e20-910c-64a7f135ffd0", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_113250_87d67a4e.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 2493.109, "limit_mb": 8796093022207.996}, "detail": {"pid": 272, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:32:50+09:00", "pid": 272, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.941, "name": "ffmpeg", "pid": 272, "rss_mb": 57.941, "state": "S (sleeping)", "threads": 1}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.905}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "warning", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["mjpeg_age_sec>10s"]}, "instance_id": "87d67a4e", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 29.049, "exists": true, "mtime": "2026-07-01T11:32:23.822813+09:00", "mtime_epoch": 1782873143.8228128, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221771}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:32:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 467.293, "name": "celery", "pid": 267, "rss_mb": 467.293, "state": "R (running)", "threads": 6}, "reason_codes": ["mjpeg_output_stale_warning"], "recommended_action": "degrade_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:32:50+09:00", "state": "running", "task_id": "4b312218-a679-4a4b-8c14-d4ec265f4513", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 2772.316, "limit_mb": 8796093022207.996}, "detail": {"pid": 277, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:32:52+09:00", "pid": 277, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.84, "name": "ffmpeg", "pid": 277, "rss_mb": 136.84, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.907}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 30.421, "exists": true, "mtime": "2026-07-01T11:32:23.850813+09:00", "mtime_epoch": 1782873143.8508132, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177656}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:32:54+09:00", "process": {"exists": true, "fd_count": 106, "hwm_mb": 467.328, "name": "celery", "pid": 237, "rss_mb": 467.328, "state": "R (running)", "threads": 6}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_113334_ac75523f.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 3854.152, "limit_mb": 8796093022207.996}, "detail": {"pid": 722, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:33:34+09:00", "pid": 722, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.383, "name": "ffmpeg", "pid": 722, "rss_mb": 137.383, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.749}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "ac75523f", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 373.663, "exists": true, "mtime": "2026-07-01T11:27:23.309594+09:00", "mtime_epoch": 1782872843.309594, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 272063}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T11:33:36+09:00", "process": {"exists": true, "fd_count": 110, "hwm_mb": 470.488, "name": "celery", "pid": 239, "rss_mb": 470.488, "state": "R (running)", "threads": 7}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:33:34+09:00", "state": "running", "task_id": "c834c962-0da3-450b-98a4-97ec2da6caab", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 3}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3983.676, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ed20347ac0] error while decoding MB 3 30, bytestream -20", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:34:42+09:00", "last_error_summary": "[h264 @ 0x55ed20347ac0] error while decoding MB 3 30, bytestream -20", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 110.494, "last_started_at": "2026-07-01T11:32:52+09:00", "pid": 277, "process": {"exists": false, "pid": 277}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 288.302, "sample_count": 438, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 189.413}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 438, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:34:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.047, "last_frame_pushed_at": "2026-07-01T11:34:42+09:00", "last_frame_read_age_sec": 0.039, "last_frame_read_at": "2026-07-01T11:34:42+09:00", "mjpeg": {"age_sec": 0.032, "exists": true, "mtime": "2026-07-01T11:34:42.733397+09:00", "mtime_epoch": 1782873282.733397, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176633}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:34:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 850.895, "name": "celery", "pid": 237, "rss_mb": 845.926, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.288, "pushed": 4.288, "read": 4.288}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.126, "count": 128, "max_ms": 14.268, "p95_ms": 10.939}, "gstreamer_push": {"avg_ms": 0.878, "count": 128, "max_ms": 1.84, "p95_ms": 0.928}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.094, "count": 128, "max_ms": 45.137, "p95_ms": 42.32}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.158, "count": 128, "max_ms": 103.667, "p95_ms": 94.405}, "yolo11_640_fire_11class": {"avg_ms": 48.665, "count": 128, "max_ms": 63.62, "p95_ms": 57.926}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.855, "count": 128, "max_ms": 100.078, "p95_ms": 85.503}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.671, "count": 128, "max_ms": 297.886, "p95_ms": 257.281}, "read_interval": {"avg_ms": 234.786, "count": 128, "max_ms": 308.655, "p95_ms": 266.605}, "triton_wait": {"avg_ms": 72.124, "count": 128, "max_ms": 103.678, "p95_ms": 94.418}, "visualize": {"avg_ms": 10.388, "count": 128, "max_ms": 13.359, "p95_ms": 12.01}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.851, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 3}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3985.141, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:34:42+09:00", "last_error_summary": "[h264 @ 0x55ed20347ac0] error while decoding MB 3 30, bytestream -20", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:34:42+09:00", "last_error_summary": "[h264 @ 0x55ed20347ac0] error while decoding MB 3 30, bytestream -20", "last_exit_at": "2026-07-01T11:34:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.69, "last_started_at": "2026-07-01T11:32:52+09:00", "pid": 277, "process": {"exists": false, "pid": 277}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 288.302, "sample_count": 439, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 189.218}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:34:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T11:34:42+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T11:34:42+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T11:34:42.733397+09:00", "mtime_epoch": 1782873282.733397, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176633}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:34:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 850.895, "name": "celery", "pid": 237, "rss_mb": 845.949, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.296, "pushed": 4.296, "read": 4.262}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.15, "count": 128, "max_ms": 14.268, "p95_ms": 11.201}, "gstreamer_push": {"avg_ms": 0.878, "count": 128, "max_ms": 1.84, "p95_ms": 0.928}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.046, "count": 128, "max_ms": 45.137, "p95_ms": 42.32}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.145, "count": 128, "max_ms": 103.667, "p95_ms": 94.405}, "yolo11_640_fire_11class": {"avg_ms": 48.61, "count": 128, "max_ms": 63.62, "p95_ms": 57.782}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.772, "count": 128, "max_ms": 100.078, "p95_ms": 85.503}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.586, "count": 128, "max_ms": 297.886, "p95_ms": 257.281}, "read_interval": {"avg_ms": 234.67, "count": 127, "max_ms": 308.655, "p95_ms": 266.605}, "triton_wait": {"avg_ms": 72.079, "count": 128, "max_ms": 103.678, "p95_ms": 94.418}, "visualize": {"avg_ms": 10.375, "count": 128, "max_ms": 13.359, "p95_ms": 12.01}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.795, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 3}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3985.027, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:34:42+09:00", "last_error_summary": "[h264 @ 0x55ed20347ac0] error while decoding MB 3 30, bytestream -20", "last_exit_at": "2026-07-01T11:34:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.693, "last_started_at": "2026-07-01T11:32:52+09:00", "pid": 277, "process": {"exists": false, "pid": 277}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 288.302, "sample_count": 439, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 189.215}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:34:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T11:34:42+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T11:34:42+09:00", "mjpeg": {"age_sec": 0.23, "exists": true, "mtime": "2026-07-01T11:34:42.733397+09:00", "mtime_epoch": 1782873282.733397, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176633}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:34:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 850.895, "name": "celery", "pid": 237, "rss_mb": 845.949, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.296, "pushed": 4.296, "read": 4.262}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.15, "count": 128, "max_ms": 14.268, "p95_ms": 11.201}, "gstreamer_push": {"avg_ms": 0.878, "count": 128, "max_ms": 1.84, "p95_ms": 0.928}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.046, "count": 128, "max_ms": 45.137, "p95_ms": 42.32}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.145, "count": 128, "max_ms": 103.667, "p95_ms": 94.405}, "yolo11_640_fire_11class": {"avg_ms": 48.61, "count": 128, "max_ms": 63.62, "p95_ms": 57.782}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.772, "count": 128, "max_ms": 100.078, "p95_ms": 85.503}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.586, "count": 128, "max_ms": 297.886, "p95_ms": 257.281}, "read_interval": {"avg_ms": 234.67, "count": 127, "max_ms": 308.655, "p95_ms": 266.605}, "triton_wait": {"avg_ms": 72.079, "count": 128, "max_ms": 103.678, "p95_ms": 94.418}, "visualize": {"avg_ms": 10.375, "count": 128, "max_ms": 13.359, "p95_ms": 12.01}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.798, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 3}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4092.258, "limit_mb": 8796093022207.996}, "detail": {"pid": 989, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:34:42+09:00", "last_error_summary": "[h264 @ 0x55ed20347ac0] error while decoding MB 3 30, bytestream -20", "last_exit_at": "2026-07-01T11:34:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:34:45+09:00", "pid": 989, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.254, "name": "ffmpeg", "pid": 989, "rss_mb": 139.254, "state": "S (sleeping)", "threads": 106}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 288.302, "sample_count": 439, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 184.252}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:34:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 4.968, "last_frame_pushed_at": "2026-07-01T11:34:42+09:00", "last_frame_read_age_sec": 5.2, "last_frame_read_at": "2026-07-01T11:34:42+09:00", "mjpeg": {"age_sec": 4.952, "exists": true, "mtime": "2026-07-01T11:34:42.973402+09:00", "mtime_epoch": 1782873282.9734015, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173890}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:34:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 850.895, "name": "celery", "pid": 237, "rss_mb": 845.949, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.588, "pushed": 3.588, "read": 3.554}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.168, "count": 107, "max_ms": 14.268, "p95_ms": 11.389}, "gstreamer_push": {"avg_ms": 0.884, "count": 107, "max_ms": 1.84, "p95_ms": 0.936}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.327, "count": 107, "max_ms": 45.137, "p95_ms": 42.465}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.362, "count": 107, "max_ms": 103.667, "p95_ms": 94.405}, "yolo11_640_fire_11class": {"avg_ms": 48.5, "count": 107, "max_ms": 63.62, "p95_ms": 57.926}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.658, "count": 107, "max_ms": 100.078, "p95_ms": 85.191}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.183, "count": 107, "max_ms": 297.886, "p95_ms": 257.281}, "read_interval": {"avg_ms": 234.277, "count": 106, "max_ms": 308.655, "p95_ms": 266.605}, "triton_wait": {"avg_ms": 72.105, "count": 107, "max_ms": 103.678, "p95_ms": 94.418}, "visualize": {"avg_ms": 10.353, "count": 107, "max_ms": 13.359, "p95_ms": 12.095}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.823, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4672.828, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55896aaeaa40] error while decoding MB 65 10, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:36:48+09:00", "last_error_summary": "[h264 @ 0x55896aaeaa40] error while decoding MB 65 10, bytestream -7", "last_exit_at": "2026-07-01T11:34:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.795, "last_started_at": "2026-07-01T11:34:45+09:00", "pid": 989, "process": {"exists": false, "pid": 989}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 403.692, "sample_count": 788, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 63.458}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 788, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:36:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.053, "last_frame_pushed_at": "2026-07-01T11:36:48+09:00", "last_frame_read_age_sec": 0.046, "last_frame_read_at": "2026-07-01T11:36:48+09:00", "mjpeg": {"age_sec": 0.036, "exists": true, "mtime": "2026-07-01T11:36:48.683689+09:00", "mtime_epoch": 1782873408.6836889, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:36:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1480.262, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 95, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.183, "pushed": 3.183, "read": 3.183}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.208, "count": 94, "max_ms": 16.645, "p95_ms": 15.478}, "gstreamer_push": {"avg_ms": 0.888, "count": 95, "max_ms": 1.415, "p95_ms": 0.989}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.327, "count": 94, "max_ms": 40.955, "p95_ms": 36.439}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.46, "count": 94, "max_ms": 98.031, "p95_ms": 96.291}, "yolo11_640_fire_11class": {"avg_ms": 47.033, "count": 94, "max_ms": 62.074, "p95_ms": 58.416}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.072, "count": 94, "max_ms": 87.039, "p95_ms": 83.145}}, "truncated_count": 0}, "processing_total": {"avg_ms": 307.344, "count": 95, "max_ms": 377.623, "p95_ms": 340.976}, "read_interval": {"avg_ms": 316.803, "count": 95, "max_ms": 387.688, "p95_ms": 350.723}, "triton_wait": {"avg_ms": 72.754, "count": 94, "max_ms": 98.044, "p95_ms": 96.304}, "visualize": {"avg_ms": 13.553, "count": 94, "max_ms": 17.418, "p95_ms": 15.628}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.847, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4671.855, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:36:48+09:00", "last_error_summary": "[h264 @ 0x55896aaeaa40] error while decoding MB 65 10, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:36:48+09:00", "last_error_summary": "[h264 @ 0x55896aaeaa40] error while decoding MB 65 10, bytestream -7", "last_exit_at": "2026-07-01T11:36:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.034, "last_started_at": "2026-07-01T11:34:45+09:00", "pid": 989, "process": {"exists": false, "pid": 989}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 403.692, "sample_count": 789, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 63.22}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:36:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-01T11:36:48+09:00", "last_frame_read_age_sec": 0.284, "last_frame_read_at": "2026-07-01T11:36:48+09:00", "mjpeg": {"age_sec": 0.275, "exists": true, "mtime": "2026-07-01T11:36:48.683689+09:00", "mtime_epoch": 1782873408.6836889, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:36:48+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1480.273, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.192, "pushed": 3.192, "read": 3.159}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.197, "count": 95, "max_ms": 16.645, "p95_ms": 15.478}, "gstreamer_push": {"avg_ms": 0.887, "count": 95, "max_ms": 1.415, "p95_ms": 0.989}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.318, "count": 95, "max_ms": 40.955, "p95_ms": 36.439}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.44, "count": 95, "max_ms": 98.031, "p95_ms": 96.291}, "yolo11_640_fire_11class": {"avg_ms": 47.082, "count": 95, "max_ms": 62.074, "p95_ms": 58.416}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.078, "count": 95, "max_ms": 87.039, "p95_ms": 83.145}}, "truncated_count": 0}, "processing_total": {"avg_ms": 307.188, "count": 95, "max_ms": 377.623, "p95_ms": 340.976}, "read_interval": {"avg_ms": 316.937, "count": 94, "max_ms": 387.688, "p95_ms": 350.723}, "triton_wait": {"avg_ms": 72.73, "count": 95, "max_ms": 98.044, "p95_ms": 96.304}, "visualize": {"avg_ms": 13.541, "count": 95, "max_ms": 17.418, "p95_ms": 15.628}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.758, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4671.848, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:36:48+09:00", "last_error_summary": "[h264 @ 0x55896aaeaa40] error while decoding MB 65 10, bytestream -7", "last_exit_at": "2026-07-01T11:36:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.037, "last_started_at": "2026-07-01T11:34:45+09:00", "pid": 989, "process": {"exists": false, "pid": 989}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 403.692, "sample_count": 789, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 63.218}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:36:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T11:36:48+09:00", "last_frame_read_age_sec": 0.287, "last_frame_read_at": "2026-07-01T11:36:48+09:00", "mjpeg": {"age_sec": 0.278, "exists": true, "mtime": "2026-07-01T11:36:48.683689+09:00", "mtime_epoch": 1782873408.6836889, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:36:48+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1480.273, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.192, "pushed": 3.192, "read": 3.159}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.197, "count": 95, "max_ms": 16.645, "p95_ms": 15.478}, "gstreamer_push": {"avg_ms": 0.887, "count": 95, "max_ms": 1.415, "p95_ms": 0.989}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.318, "count": 95, "max_ms": 40.955, "p95_ms": 36.439}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.44, "count": 95, "max_ms": 98.031, "p95_ms": 96.291}, "yolo11_640_fire_11class": {"avg_ms": 47.082, "count": 95, "max_ms": 62.074, "p95_ms": 58.416}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.078, "count": 95, "max_ms": 87.039, "p95_ms": 83.145}}, "truncated_count": 0}, "processing_total": {"avg_ms": 307.188, "count": 95, "max_ms": 377.623, "p95_ms": 340.976}, "read_interval": {"avg_ms": 316.937, "count": 94, "max_ms": 387.688, "p95_ms": 350.723}, "triton_wait": {"avg_ms": 72.73, "count": 95, "max_ms": 98.044, "p95_ms": 96.304}, "visualize": {"avg_ms": 13.541, "count": 95, "max_ms": 17.418, "p95_ms": 15.628}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.761, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4778.871, "limit_mb": 8796093022207.996}, "detail": {"pid": 1212, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:36:48+09:00", "last_error_summary": "[h264 @ 0x55896aaeaa40] error while decoding MB 65 10, bytestream -7", "last_exit_at": "2026-07-01T11:36:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:36:51+09:00", "pid": 1212, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.016, "name": "ffmpeg", "pid": 1212, "rss_mb": 137.016, "state": "S (sleeping)", "threads": 106}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 403.692, "sample_count": 789, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 58.283}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:36:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 4.939, "last_frame_pushed_at": "2026-07-01T11:36:48+09:00", "last_frame_read_age_sec": 5.221, "last_frame_read_at": "2026-07-01T11:36:48+09:00", "mjpeg": {"age_sec": 4.919, "exists": true, "mtime": "2026-07-01T11:36:48.975694+09:00", "mtime_epoch": 1782873408.9756942, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:36:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1480.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 79, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.68, "pushed": 2.68, "read": 2.646}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.983, "count": 79, "max_ms": 16.645, "p95_ms": 15.461}, "gstreamer_push": {"avg_ms": 0.887, "count": 80, "max_ms": 1.415, "p95_ms": 0.989}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.098, "count": 79, "max_ms": 39.979, "p95_ms": 35.585}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.093, "count": 79, "max_ms": 98.031, "p95_ms": 96.588}, "yolo11_640_fire_11class": {"avg_ms": 47.072, "count": 79, "max_ms": 59.294, "p95_ms": 55.361}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.451, "count": 79, "max_ms": 87.039, "p95_ms": 84.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 306.127, "count": 80, "max_ms": 354.354, "p95_ms": 336.054}, "read_interval": {"avg_ms": 315.894, "count": 79, "max_ms": 363.803, "p95_ms": 350.723}, "triton_wait": {"avg_ms": 72.329, "count": 79, "max_ms": 98.044, "p95_ms": 96.6}, "visualize": {"avg_ms": 13.566, "count": 79, "max_ms": 17.418, "p95_ms": 15.628}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.851, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_113250_87d67a4e.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4703.461, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5586bee8cb40] error while decoding MB 47 53, bytestream -19 [in#0/rtsp @ 0x5586bedbae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5586bedbae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:37:03+09:00", "last_error_summary": "[h264 @ 0x5586bee8cb40] error while decoding MB 47 53, bytestream -19 [in#0/rtsp @ 0x5586bedbae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5586bedbae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 252.495, "last_started_at": "2026-07-01T11:32:50+09:00", "pid": 272, "process": {"exists": false, "pid": 272}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.941, "sample_count": 2081, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 47.41}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:37:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "87d67a4e", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T11:37:03+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-01T11:37:03+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-01T11:37:03.311953+09:00", "mtime_epoch": 1782873423.3119526, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228460}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:37:03+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 863.512, "name": "celery", "pid": 267, "rss_mb": 858.359, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.812, "pushed": 0.812, "read": 0.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.625, "count": 14, "max_ms": 2.117, "p95_ms": 2.117}, "gstreamer_push": {"avg_ms": 1.436, "count": 14, "max_ms": 3.529, "p95_ms": 3.529}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.815, "count": 14, "max_ms": 50.304, "p95_ms": 50.304}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.774, "count": 14, "max_ms": 77.929, "p95_ms": 77.929}, "yolo11_640_fire_11class": {"avg_ms": 41.87, "count": 14, "max_ms": 50.427, "p95_ms": 50.427}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 41.629, "count": 14, "max_ms": 47.223, "p95_ms": 47.223}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.033, "count": 14, "max_ms": 88.605, "p95_ms": 88.605}, "read_interval": {"avg_ms": 2153.602, "count": 14, "max_ms": 16273.571, "p95_ms": 16273.571}, "triton_wait": {"avg_ms": 49.836, "count": 14, "max_ms": 77.941, "p95_ms": 77.941}, "visualize": {"avg_ms": 3.764, "count": 14, "max_ms": 4.234, "p95_ms": 4.234}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.235, "window_s": 30.0}, "started_at": "2026-07-01T11:32:50+09:00", "state": "running", "task_id": "4b312218-a679-4a4b-8c14-d4ec265f4513", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_113250_87d67a4e.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4703.188, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:37:03+09:00", "last_error_summary": "[h264 @ 0x5586bee8cb40] error while decoding MB 47 53, bytestream -19 [in#0/rtsp @ 0x5586bedbae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5586bedbae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:37:03+09:00", "last_error_summary": "[h264 @ 0x5586bee8cb40] error while decoding MB 47 53, bytestream -19 [in#0/rtsp @ 0x5586bedbae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5586bedbae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:37:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 252.505, "last_started_at": "2026-07-01T11:32:50+09:00", "pid": 272, "process": {"exists": false, "pid": 272}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.941, "sample_count": 2081, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 47.402}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:37:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "87d67a4e", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T11:37:03+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T11:37:03+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T11:37:03.311953+09:00", "mtime_epoch": 1782873423.3119526, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228460}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:37:03+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 863.512, "name": "celery", "pid": 267, "rss_mb": 858.398, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.812, "pushed": 0.812, "read": 0.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.625, "count": 14, "max_ms": 2.117, "p95_ms": 2.117}, "gstreamer_push": {"avg_ms": 1.436, "count": 14, "max_ms": 3.529, "p95_ms": 3.529}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.815, "count": 14, "max_ms": 50.304, "p95_ms": 50.304}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.774, "count": 14, "max_ms": 77.929, "p95_ms": 77.929}, "yolo11_640_fire_11class": {"avg_ms": 41.87, "count": 14, "max_ms": 50.427, "p95_ms": 50.427}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 41.629, "count": 14, "max_ms": 47.223, "p95_ms": 47.223}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.033, "count": 14, "max_ms": 88.605, "p95_ms": 88.605}, "read_interval": {"avg_ms": 2153.602, "count": 14, "max_ms": 16273.571, "p95_ms": 16273.571}, "triton_wait": {"avg_ms": 49.836, "count": 14, "max_ms": 77.941, "p95_ms": 77.941}, "visualize": {"avg_ms": 3.764, "count": 14, "max_ms": 4.234, "p95_ms": 4.234}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.243, "window_s": 30.0}, "started_at": "2026-07-01T11:32:50+09:00", "state": "running", "task_id": "4b312218-a679-4a4b-8c14-d4ec265f4513", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_113250_87d67a4e.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4703.012, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:37:03+09:00", "last_error_summary": "[h264 @ 0x5586bee8cb40] error while decoding MB 47 53, bytestream -19 [in#0/rtsp @ 0x5586bedbae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5586bedbae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:37:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 252.513, "last_started_at": "2026-07-01T11:32:50+09:00", "pid": 272, "process": {"exists": false, "pid": 272}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.941, "sample_count": 2081, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 47.394}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:37:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "87d67a4e", "last_frame_pushed_age_sec": 0.021, "last_frame_pushed_at": "2026-07-01T11:37:03+09:00", "last_frame_read_age_sec": 0.076, "last_frame_read_at": "2026-07-01T11:37:03+09:00", "mjpeg": {"age_sec": 0.007, "exists": true, "mtime": "2026-07-01T11:37:03.375954+09:00", "mtime_epoch": 1782873423.3759537, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 229229}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:37:03+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 863.512, "name": "celery", "pid": 267, "rss_mb": 858.398, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.812, "pushed": 0.812, "read": 0.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.625, "count": 14, "max_ms": 2.117, "p95_ms": 2.117}, "gstreamer_push": {"avg_ms": 1.436, "count": 14, "max_ms": 3.529, "p95_ms": 3.529}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.815, "count": 14, "max_ms": 50.304, "p95_ms": 50.304}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.774, "count": 14, "max_ms": 77.929, "p95_ms": 77.929}, "yolo11_640_fire_11class": {"avg_ms": 41.87, "count": 14, "max_ms": 50.427, "p95_ms": 50.427}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 41.629, "count": 14, "max_ms": 47.223, "p95_ms": 47.223}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.033, "count": 14, "max_ms": 88.605, "p95_ms": 88.605}, "read_interval": {"avg_ms": 2153.602, "count": 14, "max_ms": 16273.571, "p95_ms": 16273.571}, "triton_wait": {"avg_ms": 49.836, "count": 14, "max_ms": 77.941, "p95_ms": 77.941}, "visualize": {"avg_ms": 3.764, "count": 14, "max_ms": 4.234, "p95_ms": 4.234}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.251, "window_s": 30.0}, "started_at": "2026-07-01T11:32:50+09:00", "state": "running", "task_id": "4b312218-a679-4a4b-8c14-d4ec265f4513", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_113250_87d67a4e.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4861.484, "limit_mb": 8796093022207.996}, "detail": {"pid": 1379, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:37:03+09:00", "last_error_summary": "[h264 @ 0x5586bee8cb40] error while decoding MB 47 53, bytestream -19 [in#0/rtsp @ 0x5586bedbae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5586bedbae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:37:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:37:05+09:00", "pid": 1379, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.727, "name": "ffmpeg", "pid": 1379, "rss_mb": 156.562, "state": "R (running)", "threads": 49}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.941, "sample_count": 2081, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 43.305}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:37:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "87d67a4e", "last_frame_pushed_age_sec": 4.111, "last_frame_pushed_at": "2026-07-01T11:37:03+09:00", "last_frame_read_age_sec": 4.165, "last_frame_read_at": "2026-07-01T11:37:03+09:00", "mjpeg": {"age_sec": 4.096, "exists": true, "mtime": "2026-07-01T11:37:03.375954+09:00", "mtime_epoch": 1782873423.3759537, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 229229}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:37:07+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 863.512, "name": "celery", "pid": 267, "rss_mb": 858.398, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.656, "pushed": 0.656, "read": 0.656}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.625, "count": 14, "max_ms": 2.117, "p95_ms": 2.117}, "gstreamer_push": {"avg_ms": 1.436, "count": 14, "max_ms": 3.529, "p95_ms": 3.529}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.815, "count": 14, "max_ms": 50.304, "p95_ms": 50.304}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.774, "count": 14, "max_ms": 77.929, "p95_ms": 77.929}, "yolo11_640_fire_11class": {"avg_ms": 41.87, "count": 14, "max_ms": 50.427, "p95_ms": 50.427}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 41.629, "count": 14, "max_ms": 47.223, "p95_ms": 47.223}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.033, "count": 14, "max_ms": 88.605, "p95_ms": 88.605}, "read_interval": {"avg_ms": 2153.602, "count": 14, "max_ms": 16273.571, "p95_ms": 16273.571}, "triton_wait": {"avg_ms": 49.836, "count": 14, "max_ms": 77.941, "p95_ms": 77.941}, "visualize": {"avg_ms": 3.764, "count": 14, "max_ms": 4.234, "p95_ms": 4.234}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.34, "window_s": 30.0}, "started_at": "2026-07-01T11:32:50+09:00", "state": "running", "task_id": "4b312218-a679-4a4b-8c14-d4ec265f4513", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 9}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4859.273, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c303dd7240] error while decoding MB 10 35, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:38:50+09:00", "last_error_summary": "[h264 @ 0x55c303dd7240] error while decoding MB 10 35, bytestream -10", "last_exit_at": "2026-07-01T11:36:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.103, "last_started_at": "2026-07-01T11:36:51+09:00", "pid": 1212, "process": {"exists": false, "pid": 1212}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 395.324, "sample_count": 946, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1170, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:38:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.044, "last_frame_pushed_at": "2026-07-01T11:38:50+09:00", "last_frame_read_age_sec": 0.037, "last_frame_read_at": "2026-07-01T11:38:50+09:00", "mjpeg": {"age_sec": 0.024, "exists": true, "mtime": "2026-07-01T11:38:50.973878+09:00", "mtime_epoch": 1782873530.973878, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183548}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:38:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.306, "pushed": 3.306, "read": 3.306}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.449, "count": 98, "max_ms": 22.184, "p95_ms": 17.658}, "gstreamer_push": {"avg_ms": 0.865, "count": 99, "max_ms": 1.148, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.379, "count": 98, "max_ms": 48.196, "p95_ms": 42.624}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.014, "count": 98, "max_ms": 112.458, "p95_ms": 92.998}, "yolo11_640_fire_11class": {"avg_ms": 45.989, "count": 98, "max_ms": 63.509, "p95_ms": 56.765}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.518, "count": 98, "max_ms": 95.314, "p95_ms": 83.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 295.777, "count": 99, "max_ms": 380.848, "p95_ms": 340.024}, "read_interval": {"avg_ms": 305.206, "count": 99, "max_ms": 389.802, "p95_ms": 351.146}, "triton_wait": {"avg_ms": 70.108, "count": 98, "max_ms": 112.474, "p95_ms": 94.423}, "visualize": {"avg_ms": 13.273, "count": 98, "max_ms": 18.941, "p95_ms": 16.553}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 9}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4860.328, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:38:50+09:00", "last_error_summary": "[h264 @ 0x55c303dd7240] error while decoding MB 10 35, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:38:50+09:00", "last_error_summary": "[h264 @ 0x55c303dd7240] error while decoding MB 10 35, bytestream -10", "last_exit_at": "2026-07-01T11:38:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.367, "last_started_at": "2026-07-01T11:36:51+09:00", "pid": 1212, "process": {"exists": false, "pid": 1212}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 395.324, "sample_count": 946, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1171, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:38:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:38:51+09:00", "last_frame_read_age_sec": 0.3, "last_frame_read_at": "2026-07-01T11:38:50+09:00", "mjpeg": {"age_sec": 0.287, "exists": true, "mtime": "2026-07-01T11:38:50.973878+09:00", "mtime_epoch": 1782873530.973878, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183548}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:38:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.315, "pushed": 3.315, "read": 3.281}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.434, "count": 98, "max_ms": 22.184, "p95_ms": 17.658}, "gstreamer_push": {"avg_ms": 0.865, "count": 99, "max_ms": 1.148, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.419, "count": 98, "max_ms": 48.196, "p95_ms": 42.624}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.832, "count": 98, "max_ms": 112.458, "p95_ms": 92.998}, "yolo11_640_fire_11class": {"avg_ms": 46.0, "count": 98, "max_ms": 63.509, "p95_ms": 56.765}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.505, "count": 98, "max_ms": 95.314, "p95_ms": 83.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 295.636, "count": 99, "max_ms": 380.848, "p95_ms": 340.024}, "read_interval": {"avg_ms": 305.078, "count": 98, "max_ms": 389.802, "p95_ms": 351.146}, "triton_wait": {"avg_ms": 69.925, "count": 98, "max_ms": 112.474, "p95_ms": 94.423}, "visualize": {"avg_ms": 13.269, "count": 98, "max_ms": 18.941, "p95_ms": 16.553}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 9}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4860.32, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:38:50+09:00", "last_error_summary": "[h264 @ 0x55c303dd7240] error while decoding MB 10 35, bytestream -10", "last_exit_at": "2026-07-01T11:38:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.37, "last_started_at": "2026-07-01T11:36:51+09:00", "pid": 1212, "process": {"exists": false, "pid": 1212}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 395.324, "sample_count": 946, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1171, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:38:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T11:38:51+09:00", "last_frame_read_age_sec": 0.304, "last_frame_read_at": "2026-07-01T11:38:50+09:00", "mjpeg": {"age_sec": 0.29, "exists": true, "mtime": "2026-07-01T11:38:50.973878+09:00", "mtime_epoch": 1782873530.973878, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183548}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:38:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.314, "pushed": 3.314, "read": 3.281}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.434, "count": 98, "max_ms": 22.184, "p95_ms": 17.658}, "gstreamer_push": {"avg_ms": 0.865, "count": 99, "max_ms": 1.148, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.419, "count": 98, "max_ms": 48.196, "p95_ms": 42.624}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.832, "count": 98, "max_ms": 112.458, "p95_ms": 92.998}, "yolo11_640_fire_11class": {"avg_ms": 46.0, "count": 98, "max_ms": 63.509, "p95_ms": 56.765}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.505, "count": 98, "max_ms": 95.314, "p95_ms": 83.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 295.636, "count": 99, "max_ms": 380.848, "p95_ms": 340.024}, "read_interval": {"avg_ms": 305.078, "count": 98, "max_ms": 389.802, "p95_ms": 351.146}, "triton_wait": {"avg_ms": 69.925, "count": 98, "max_ms": 112.474, "p95_ms": 94.423}, "visualize": {"avg_ms": 13.269, "count": 98, "max_ms": 18.941, "p95_ms": 16.553}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.872, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 9}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4961.547, "limit_mb": 8796093022207.996}, "detail": {"pid": 1592, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:38:50+09:00", "last_error_summary": "[h264 @ 0x55c303dd7240] error while decoding MB 10 35, bytestream -10", "last_exit_at": "2026-07-01T11:38:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:38:53+09:00", "pid": 1592, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.621, "name": "ffmpeg", "pid": 1592, "rss_mb": 138.621, "state": "S (sleeping)", "threads": 106}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 396.137, "sample_count": 927, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1171, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:38:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 4.73, "last_frame_pushed_at": "2026-07-01T11:38:51+09:00", "last_frame_read_age_sec": 5.027, "last_frame_read_at": "2026-07-01T11:38:50+09:00", "mjpeg": {"age_sec": 4.71, "exists": true, "mtime": "2026-07-01T11:38:51.277883+09:00", "mtime_epoch": 1782873531.2778833, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183763}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:38:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 83, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.817, "pushed": 2.817, "read": 2.783}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.452, "count": 83, "max_ms": 22.184, "p95_ms": 17.232}, "gstreamer_push": {"avg_ms": 0.865, "count": 84, "max_ms": 1.148, "p95_ms": 0.936}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.566, "count": 83, "max_ms": 48.196, "p95_ms": 42.16}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.244, "count": 83, "max_ms": 99.685, "p95_ms": 92.997}, "yolo11_640_fire_11class": {"avg_ms": 45.772, "count": 83, "max_ms": 62.599, "p95_ms": 56.274}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.56, "count": 83, "max_ms": 94.41, "p95_ms": 82.763}}, "truncated_count": 0}, "processing_total": {"avg_ms": 293.517, "count": 84, "max_ms": 380.848, "p95_ms": 338.802}, "read_interval": {"avg_ms": 302.853, "count": 83, "max_ms": 389.802, "p95_ms": 347.041}, "triton_wait": {"avg_ms": 69.534, "count": 83, "max_ms": 99.696, "p95_ms": 93.014}, "visualize": {"avg_ms": 13.258, "count": 83, "max_ms": 18.327, "p95_ms": 15.94}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.822, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4680.582, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a207b2f200] error while decoding MB 64 36, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:40:43+09:00", "last_error_summary": "[h264 @ 0x55a207b2f200] error while decoding MB 64 36, bytestream -5", "last_exit_at": "2026-07-01T11:38:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.928, "last_started_at": "2026-07-01T11:38:53+09:00", "pid": 1592, "process": {"exists": false, "pid": 1592}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 355.129, "sample_count": 1031, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1625, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:40:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T11:40:43+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T11:40:43+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T11:40:43.875874+09:00", "mtime_epoch": 1782873643.8758738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173678}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:40:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.551, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.21, "pushed": 4.21, "read": 4.21}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.496, "count": 125, "max_ms": 14.569, "p95_ms": 11.597}, "gstreamer_push": {"avg_ms": 0.884, "count": 126, "max_ms": 1.631, "p95_ms": 0.946}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.723, "count": 125, "max_ms": 56.971, "p95_ms": 46.696}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.543, "count": 125, "max_ms": 108.915, "p95_ms": 98.513}, "yolo11_640_fire_11class": {"avg_ms": 47.082, "count": 125, "max_ms": 67.915, "p95_ms": 57.74}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.264, "count": 125, "max_ms": 101.06, "p95_ms": 88.722}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.466, "count": 126, "max_ms": 269.001, "p95_ms": 256.889}, "read_interval": {"avg_ms": 239.062, "count": 126, "max_ms": 281.891, "p95_ms": 267.403}, "triton_wait": {"avg_ms": 73.904, "count": 125, "max_ms": 108.928, "p95_ms": 99.791}, "visualize": {"avg_ms": 10.547, "count": 125, "max_ms": 14.195, "p95_ms": 12.229}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4681.801, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:40:43+09:00", "last_error_summary": "[h264 @ 0x55a207b2f200] error while decoding MB 64 36, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:40:43+09:00", "last_error_summary": "[h264 @ 0x55a207b2f200] error while decoding MB 64 36, bytestream -5", "last_exit_at": "2026-07-01T11:40:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.112, "last_started_at": "2026-07-01T11:38:53+09:00", "pid": 1592, "process": {"exists": false, "pid": 1592}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 355.129, "sample_count": 1032, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1626, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:40:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:40:44+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T11:40:43+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T11:40:43.875874+09:00", "mtime_epoch": 1782873643.8758738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173678}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:40:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.216, "pushed": 4.216, "read": 4.182}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.473, "count": 125, "max_ms": 14.569, "p95_ms": 11.306}, "gstreamer_push": {"avg_ms": 0.883, "count": 126, "max_ms": 1.631, "p95_ms": 0.946}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.7, "count": 125, "max_ms": 56.971, "p95_ms": 46.696}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.656, "count": 125, "max_ms": 108.915, "p95_ms": 98.513}, "yolo11_640_fire_11class": {"avg_ms": 47.093, "count": 125, "max_ms": 67.915, "p95_ms": 57.74}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.261, "count": 125, "max_ms": 101.06, "p95_ms": 88.722}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.368, "count": 126, "max_ms": 269.001, "p95_ms": 256.889}, "read_interval": {"avg_ms": 238.956, "count": 125, "max_ms": 281.891, "p95_ms": 267.403}, "triton_wait": {"avg_ms": 74.017, "count": 125, "max_ms": 108.928, "p95_ms": 99.791}, "visualize": {"avg_ms": 10.532, "count": 125, "max_ms": 14.195, "p95_ms": 12.199}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4681.738, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:40:43+09:00", "last_error_summary": "[h264 @ 0x55a207b2f200] error while decoding MB 64 36, bytestream -5", "last_exit_at": "2026-07-01T11:40:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.116, "last_started_at": "2026-07-01T11:38:53+09:00", "pid": 1592, "process": {"exists": false, "pid": 1592}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 355.129, "sample_count": 1032, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1626, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:40:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T11:40:44+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T11:40:43+09:00", "mjpeg": {"age_sec": 0.227, "exists": true, "mtime": "2026-07-01T11:40:43.875874+09:00", "mtime_epoch": 1782873643.8758738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173678}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:40:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.215, "pushed": 4.215, "read": 4.182}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.473, "count": 125, "max_ms": 14.569, "p95_ms": 11.306}, "gstreamer_push": {"avg_ms": 0.883, "count": 126, "max_ms": 1.631, "p95_ms": 0.946}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.7, "count": 125, "max_ms": 56.971, "p95_ms": 46.696}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.656, "count": 125, "max_ms": 108.915, "p95_ms": 98.513}, "yolo11_640_fire_11class": {"avg_ms": 47.093, "count": 125, "max_ms": 67.915, "p95_ms": 57.74}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.261, "count": 125, "max_ms": 101.06, "p95_ms": 88.722}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.368, "count": 126, "max_ms": 269.001, "p95_ms": 256.889}, "read_interval": {"avg_ms": 238.956, "count": 125, "max_ms": 281.891, "p95_ms": 267.403}, "triton_wait": {"avg_ms": 74.017, "count": 125, "max_ms": 108.928, "p95_ms": 99.791}, "visualize": {"avg_ms": 10.532, "count": 125, "max_ms": 14.195, "p95_ms": 12.199}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4783.914, "limit_mb": 8796093022207.996}, "detail": {"pid": 1699, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:40:43+09:00", "last_error_summary": "[h264 @ 0x55a207b2f200] error while decoding MB 64 36, bytestream -5", "last_exit_at": "2026-07-01T11:40:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:40:46+09:00", "pid": 1699, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.562, "name": "ffmpeg", "pid": 1699, "rss_mb": 139.562, "state": "R (running)", "threads": 106}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 352.276, "sample_count": 1020, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1626, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:40:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 4.328, "last_frame_pushed_at": "2026-07-01T11:40:44+09:00", "last_frame_read_age_sec": 4.56, "last_frame_read_at": "2026-07-01T11:40:43+09:00", "mjpeg": {"age_sec": 4.309, "exists": true, "mtime": "2026-07-01T11:40:44.115878+09:00", "mtime_epoch": 1782873644.115878, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173912}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:40:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.551, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.579, "pushed": 3.579, "read": 3.545}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.647, "count": 106, "max_ms": 14.569, "p95_ms": 11.597}, "gstreamer_push": {"avg_ms": 0.881, "count": 107, "max_ms": 1.419, "p95_ms": 0.946}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.807, "count": 106, "max_ms": 56.971, "p95_ms": 46.696}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.905, "count": 106, "max_ms": 103.135, "p95_ms": 97.443}, "yolo11_640_fire_11class": {"avg_ms": 46.761, "count": 106, "max_ms": 67.915, "p95_ms": 56.109}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.926, "count": 106, "max_ms": 101.06, "p95_ms": 88.722}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.411, "count": 107, "max_ms": 269.001, "p95_ms": 260.711}, "read_interval": {"avg_ms": 241.014, "count": 106, "max_ms": 281.891, "p95_ms": 269.924}, "triton_wait": {"avg_ms": 73.269, "count": 106, "max_ms": 103.148, "p95_ms": 97.634}, "visualize": {"avg_ms": 10.699, "count": 106, "max_ms": 14.195, "p95_ms": 12.199}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4318.867, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560d31d70d80] error while decoding MB 5 43, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:42:35+09:00", "last_error_summary": "[h264 @ 0x560d31d70d80] error while decoding MB 5 43, bytestream -14", "last_exit_at": "2026-07-01T11:40:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.48, "last_started_at": "2026-07-01T11:40:46+09:00", "pid": 1699, "process": {"exists": false, "pid": 1699}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 310.645, "sample_count": 1162, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2083, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:42:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T11:42:35+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T11:42:35+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T11:42:35.857835+09:00", "mtime_epoch": 1782873755.8578346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172975}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:42:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.551, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.039, "pushed": 4.039, "read": 4.039}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.959, "count": 120, "max_ms": 15.528, "p95_ms": 12.025}, "gstreamer_push": {"avg_ms": 0.847, "count": 121, "max_ms": 1.074, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.194, "count": 120, "max_ms": 49.009, "p95_ms": 45.753}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.1, "count": 120, "max_ms": 84.125, "p95_ms": 81.548}, "yolo11_640_fire_11class": {"avg_ms": 50.533, "count": 120, "max_ms": 64.359, "p95_ms": 61.581}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.884, "count": 120, "max_ms": 85.592, "p95_ms": 81.152}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.635, "count": 121, "max_ms": 278.805, "p95_ms": 265.893}, "read_interval": {"avg_ms": 249.176, "count": 121, "max_ms": 289.299, "p95_ms": 276.711}, "triton_wait": {"avg_ms": 71.799, "count": 120, "max_ms": 85.604, "p95_ms": 82.325}, "visualize": {"avg_ms": 11.023, "count": 120, "max_ms": 13.663, "p95_ms": 12.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4315.152, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:42:35+09:00", "last_error_summary": "[h264 @ 0x560d31d70d80] error while decoding MB 5 43, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:42:35+09:00", "last_error_summary": "[h264 @ 0x560d31d70d80] error while decoding MB 5 43, bytestream -14", "last_exit_at": "2026-07-01T11:42:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.675, "last_started_at": "2026-07-01T11:40:46+09:00", "pid": 1699, "process": {"exists": false, "pid": 1699}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 310.645, "sample_count": 1163, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2084, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:42:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T11:42:36+09:00", "last_frame_read_age_sec": 0.249, "last_frame_read_at": "2026-07-01T11:42:35+09:00", "mjpeg": {"age_sec": 0.241, "exists": true, "mtime": "2026-07-01T11:42:35.857835+09:00", "mtime_epoch": 1782873755.8578346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172975}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:42:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.555, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.046, "pushed": 4.046, "read": 4.012}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.964, "count": 120, "max_ms": 15.528, "p95_ms": 12.025}, "gstreamer_push": {"avg_ms": 0.847, "count": 121, "max_ms": 1.074, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.191, "count": 120, "max_ms": 49.009, "p95_ms": 45.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.118, "count": 120, "max_ms": 84.125, "p95_ms": 81.548}, "yolo11_640_fire_11class": {"avg_ms": 50.498, "count": 120, "max_ms": 64.359, "p95_ms": 61.578}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.912, "count": 120, "max_ms": 85.592, "p95_ms": 81.152}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.641, "count": 121, "max_ms": 278.805, "p95_ms": 265.893}, "read_interval": {"avg_ms": 249.152, "count": 120, "max_ms": 289.299, "p95_ms": 276.711}, "triton_wait": {"avg_ms": 71.817, "count": 120, "max_ms": 85.604, "p95_ms": 82.325}, "visualize": {"avg_ms": 11.02, "count": 120, "max_ms": 13.663, "p95_ms": 12.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4315.141, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:42:35+09:00", "last_error_summary": "[h264 @ 0x560d31d70d80] error while decoding MB 5 43, bytestream -14", "last_exit_at": "2026-07-01T11:42:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.678, "last_started_at": "2026-07-01T11:40:46+09:00", "pid": 1699, "process": {"exists": false, "pid": 1699}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 310.645, "sample_count": 1163, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2084, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:42:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T11:42:36+09:00", "last_frame_read_age_sec": 0.252, "last_frame_read_at": "2026-07-01T11:42:35+09:00", "mjpeg": {"age_sec": 0.244, "exists": true, "mtime": "2026-07-01T11:42:35.857835+09:00", "mtime_epoch": 1782873755.8578346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172975}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:42:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.555, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.045, "pushed": 4.045, "read": 4.012}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.964, "count": 120, "max_ms": 15.528, "p95_ms": 12.025}, "gstreamer_push": {"avg_ms": 0.847, "count": 121, "max_ms": 1.074, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.191, "count": 120, "max_ms": 49.009, "p95_ms": 45.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.118, "count": 120, "max_ms": 84.125, "p95_ms": 81.548}, "yolo11_640_fire_11class": {"avg_ms": 50.498, "count": 120, "max_ms": 64.359, "p95_ms": 61.578}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.912, "count": 120, "max_ms": 85.592, "p95_ms": 81.152}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.641, "count": 121, "max_ms": 278.805, "p95_ms": 265.893}, "read_interval": {"avg_ms": 249.152, "count": 120, "max_ms": 289.299, "p95_ms": 276.711}, "triton_wait": {"avg_ms": 71.817, "count": 120, "max_ms": 85.604, "p95_ms": 82.325}, "visualize": {"avg_ms": 11.02, "count": 120, "max_ms": 13.663, "p95_ms": 12.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4423.445, "limit_mb": 8796093022207.996}, "detail": {"pid": 1806, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:42:35+09:00", "last_error_summary": "[h264 @ 0x560d31d70d80] error while decoding MB 5 43, bytestream -14", "last_exit_at": "2026-07-01T11:42:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:42:38+09:00", "pid": 1806, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.082, "name": "ffmpeg", "pid": 1806, "rss_mb": 139.082, "state": "S (sleeping)", "threads": 106}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 310.645, "sample_count": 1146, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2084, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:42:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 4.47, "last_frame_pushed_at": "2026-07-01T11:42:36+09:00", "last_frame_read_age_sec": 4.715, "last_frame_read_at": "2026-07-01T11:42:35+09:00", "mjpeg": {"age_sec": 4.455, "exists": true, "mtime": "2026-07-01T11:42:36.109839+09:00", "mtime_epoch": 1782873756.109839, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168444}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:42:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.555, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.437, "pushed": 3.437, "read": 3.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.969, "count": 102, "max_ms": 15.528, "p95_ms": 12.025}, "gstreamer_push": {"avg_ms": 0.847, "count": 103, "max_ms": 1.074, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.814, "count": 102, "max_ms": 46.907, "p95_ms": 44.648}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.155, "count": 102, "max_ms": 84.125, "p95_ms": 82.132}, "yolo11_640_fire_11class": {"avg_ms": 50.312, "count": 102, "max_ms": 64.359, "p95_ms": 61.581}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.148, "count": 102, "max_ms": 85.592, "p95_ms": 81.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.415, "count": 103, "max_ms": 278.805, "p95_ms": 267.832}, "read_interval": {"avg_ms": 249.943, "count": 102, "max_ms": 289.299, "p95_ms": 277.63}, "triton_wait": {"avg_ms": 71.917, "count": 102, "max_ms": 85.604, "p95_ms": 82.982}, "visualize": {"avg_ms": 11.019, "count": 102, "max_ms": 13.663, "p95_ms": 12.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 21}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4320.207, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555c4431d380] error while decoding MB 52 10, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:44:30+09:00", "last_error_summary": "[h264 @ 0x555c4431d380] error while decoding MB 52 10, bytestream -5", "last_exit_at": "2026-07-01T11:42:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.423, "last_started_at": "2026-07-01T11:42:38+09:00", "pid": 1806, "process": {"exists": false, "pid": 1806}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.346, "sample_count": 1203, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2526, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:44:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T11:44:30+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T11:44:30+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T11:44:30.947834+09:00", "mtime_epoch": 1782873870.9478338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171608}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:44:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.812, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.608, "pushed": 3.608, "read": 3.608}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.572, "count": 107, "max_ms": 13.729, "p95_ms": 12.619}, "gstreamer_push": {"avg_ms": 0.854, "count": 108, "max_ms": 1.195, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.865, "count": 107, "max_ms": 45.757, "p95_ms": 42.096}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.329, "count": 107, "max_ms": 84.569, "p95_ms": 81.371}, "yolo11_640_fire_11class": {"avg_ms": 48.086, "count": 107, "max_ms": 64.778, "p95_ms": 60.288}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.957, "count": 107, "max_ms": 82.937, "p95_ms": 79.353}}, "truncated_count": 0}, "processing_total": {"avg_ms": 269.642, "count": 108, "max_ms": 294.551, "p95_ms": 283.177}, "read_interval": {"avg_ms": 279.134, "count": 108, "max_ms": 303.849, "p95_ms": 293.152}, "triton_wait": {"avg_ms": 69.666, "count": 107, "max_ms": 84.582, "p95_ms": 81.75}, "visualize": {"avg_ms": 12.495, "count": 107, "max_ms": 15.753, "p95_ms": 13.367}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 21}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4319.402, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:44:30+09:00", "last_error_summary": "[h264 @ 0x555c4431d380] error while decoding MB 52 10, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:44:30+09:00", "last_error_summary": "[h264 @ 0x555c4431d380] error while decoding MB 52 10, bytestream -5", "last_exit_at": "2026-07-01T11:44:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.632, "last_started_at": "2026-07-01T11:42:38+09:00", "pid": 1806, "process": {"exists": false, "pid": 1806}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.346, "sample_count": 1203, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2527, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:44:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:44:31+09:00", "last_frame_read_age_sec": 0.26, "last_frame_read_at": "2026-07-01T11:44:30+09:00", "mjpeg": {"age_sec": 0.248, "exists": true, "mtime": "2026-07-01T11:44:30.947834+09:00", "mtime_epoch": 1782873870.9478338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171608}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:44:31+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.812, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.612, "pushed": 3.612, "read": 3.578}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.593, "count": 107, "max_ms": 13.729, "p95_ms": 12.619}, "gstreamer_push": {"avg_ms": 0.853, "count": 108, "max_ms": 1.195, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.822, "count": 107, "max_ms": 45.757, "p95_ms": 42.096}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.268, "count": 107, "max_ms": 84.569, "p95_ms": 81.371}, "yolo11_640_fire_11class": {"avg_ms": 47.974, "count": 107, "max_ms": 64.778, "p95_ms": 60.288}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.891, "count": 107, "max_ms": 82.937, "p95_ms": 79.353}}, "truncated_count": 0}, "processing_total": {"avg_ms": 269.52, "count": 108, "max_ms": 294.551, "p95_ms": 283.177}, "read_interval": {"avg_ms": 279.148, "count": 107, "max_ms": 303.849, "p95_ms": 293.152}, "triton_wait": {"avg_ms": 69.605, "count": 107, "max_ms": 84.582, "p95_ms": 81.75}, "visualize": {"avg_ms": 12.507, "count": 107, "max_ms": 15.753, "p95_ms": 13.367}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 21}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4319.395, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:44:30+09:00", "last_error_summary": "[h264 @ 0x555c4431d380] error while decoding MB 52 10, bytestream -5", "last_exit_at": "2026-07-01T11:44:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.635, "last_started_at": "2026-07-01T11:42:38+09:00", "pid": 1806, "process": {"exists": false, "pid": 1806}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.346, "sample_count": 1203, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2527, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:44:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T11:44:31+09:00", "last_frame_read_age_sec": 0.263, "last_frame_read_at": "2026-07-01T11:44:30+09:00", "mjpeg": {"age_sec": 0.251, "exists": true, "mtime": "2026-07-01T11:44:30.947834+09:00", "mtime_epoch": 1782873870.9478338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171608}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:44:31+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.812, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.611, "pushed": 3.611, "read": 3.578}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.593, "count": 107, "max_ms": 13.729, "p95_ms": 12.619}, "gstreamer_push": {"avg_ms": 0.853, "count": 108, "max_ms": 1.195, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.822, "count": 107, "max_ms": 45.757, "p95_ms": 42.096}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.268, "count": 107, "max_ms": 84.569, "p95_ms": 81.371}, "yolo11_640_fire_11class": {"avg_ms": 47.974, "count": 107, "max_ms": 64.778, "p95_ms": 60.288}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.891, "count": 107, "max_ms": 82.937, "p95_ms": 79.353}}, "truncated_count": 0}, "processing_total": {"avg_ms": 269.52, "count": 108, "max_ms": 294.551, "p95_ms": 283.177}, "read_interval": {"avg_ms": 279.148, "count": 107, "max_ms": 303.849, "p95_ms": 293.152}, "triton_wait": {"avg_ms": 69.605, "count": 107, "max_ms": 84.582, "p95_ms": 81.75}, "visualize": {"avg_ms": 12.507, "count": 107, "max_ms": 15.753, "p95_ms": 13.367}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 21}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4425.07, "limit_mb": 8796093022207.996}, "detail": {"pid": 1913, "restart_count": 6, "start_count": 7}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:44:30+09:00", "last_error_summary": "[h264 @ 0x555c4431d380] error while decoding MB 52 10, bytestream -5", "last_exit_at": "2026-07-01T11:44:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:44:33+09:00", "pid": 1913, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.715, "name": "ffmpeg", "pid": 1913, "rss_mb": 136.715, "state": "S (sleeping)", "threads": 106}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.346, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2527, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:44:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 4.265, "last_frame_pushed_at": "2026-07-01T11:44:31+09:00", "last_frame_read_age_sec": 4.522, "last_frame_read_at": "2026-07-01T11:44:30+09:00", "mjpeg": {"age_sec": 4.246, "exists": true, "mtime": "2026-07-01T11:44:31.211838+09:00", "mtime_epoch": 1782873871.2118382, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165450}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:44:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1401.812, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 91, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.089, "pushed": 3.089, "read": 3.056}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.706, "count": 92, "max_ms": 13.729, "p95_ms": 12.755}, "gstreamer_push": {"avg_ms": 0.855, "count": 92, "max_ms": 1.195, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.682, "count": 92, "max_ms": 45.757, "p95_ms": 42.142}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.807, "count": 92, "max_ms": 84.569, "p95_ms": 81.342}, "yolo11_640_fire_11class": {"avg_ms": 47.521, "count": 92, "max_ms": 64.778, "p95_ms": 60.288}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.597, "count": 92, "max_ms": 82.937, "p95_ms": 79.427}}, "truncated_count": 0}, "processing_total": {"avg_ms": 270.856, "count": 92, "max_ms": 294.551, "p95_ms": 284.502}, "read_interval": {"avg_ms": 280.519, "count": 91, "max_ms": 303.849, "p95_ms": 293.531}, "triton_wait": {"avg_ms": 69.197, "count": 92, "max_ms": 84.582, "p95_ms": 81.75}, "visualize": {"avg_ms": 12.542, "count": 92, "max_ms": 15.753, "p95_ms": 13.655}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.782, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_113250_87d67a4e.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 49}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4252.363, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a5f4bc5000] error while decoding MB 55 42, bytestream -13 [in#0/rtsp @ 0x55a5f4b7de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a5f4b7de00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:44:54+09:00", "last_error_summary": "[h264 @ 0x55a5f4bc5000] error while decoding MB 55 42, bytestream -13 [in#0/rtsp @ 0x55a5f4b7de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a5f4b7de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:37:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 468.563, "last_started_at": "2026-07-01T11:37:05+09:00", "pid": 1379, "process": {"exists": false, "pid": 1379}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.723, "sample_count": 2782, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5972, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:44:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "87d67a4e", "last_frame_pushed_age_sec": 0.001, "last_frame_pushed_at": "2026-07-01T11:44:54+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-01T11:44:53+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-01T11:44:53.980232+09:00", "mtime_epoch": 1782873893.9802322, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222348}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:44:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1131.152, "name": "celery", "pid": 267, "rss_mb": 867.992, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.27, "pushed": 2.27, "read": 2.236}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.981, "count": 68, "max_ms": 2.463, "p95_ms": 2.103}, "gstreamer_push": {"avg_ms": 1.398, "count": 68, "max_ms": 3.828, "p95_ms": 1.975}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.547, "count": 68, "max_ms": 48.761, "p95_ms": 43.335}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.351, "count": 68, "max_ms": 71.487, "p95_ms": 52.423}, "yolo11_640_fire_11class": {"avg_ms": 41.853, "count": 68, "max_ms": 47.694, "p95_ms": 45.97}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.319, "count": 68, "max_ms": 68.317, "p95_ms": 50.276}}, "truncated_count": 0}, "processing_total": {"avg_ms": 57.552, "count": 68, "max_ms": 81.951, "p95_ms": 62.973}, "read_interval": {"avg_ms": 447.123, "count": 67, "max_ms": 14976.012, "p95_ms": 86.582}, "triton_wait": {"avg_ms": 47.396, "count": 68, "max_ms": 71.496, "p95_ms": 52.435}, "visualize": {"avg_ms": 3.468, "count": 68, "max_ms": 3.803, "p95_ms": 3.697}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T11:32:50+09:00", "state": "running", "task_id": "4b312218-a679-4a4b-8c14-d4ec265f4513", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_113250_87d67a4e.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 49}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4252.277, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:44:54+09:00", "last_error_summary": "[h264 @ 0x55a5f4bc5000] error while decoding MB 55 42, bytestream -13 [in#0/rtsp @ 0x55a5f4b7de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a5f4b7de00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:44:54+09:00", "last_error_summary": "[h264 @ 0x55a5f4bc5000] error while decoding MB 55 42, bytestream -13 [in#0/rtsp @ 0x55a5f4b7de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a5f4b7de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:44:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 468.578, "last_started_at": "2026-07-01T11:37:05+09:00", "pid": 1379, "process": {"exists": false, "pid": 1379}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.723, "sample_count": 2781, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5972, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:44:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "87d67a4e", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T11:44:54+09:00", "last_frame_read_age_sec": 0.073, "last_frame_read_at": "2026-07-01T11:44:53+09:00", "mjpeg": {"age_sec": 0.069, "exists": true, "mtime": "2026-07-01T11:44:53.980232+09:00", "mtime_epoch": 1782873893.9802322, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222348}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:44:54+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1131.152, "name": "celery", "pid": 267, "rss_mb": 868.004, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.269, "pushed": 2.269, "read": 2.235}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.981, "count": 68, "max_ms": 2.463, "p95_ms": 2.103}, "gstreamer_push": {"avg_ms": 1.398, "count": 68, "max_ms": 3.828, "p95_ms": 1.975}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.547, "count": 68, "max_ms": 48.761, "p95_ms": 43.335}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.351, "count": 68, "max_ms": 71.487, "p95_ms": 52.423}, "yolo11_640_fire_11class": {"avg_ms": 41.853, "count": 68, "max_ms": 47.694, "p95_ms": 45.97}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.319, "count": 68, "max_ms": 68.317, "p95_ms": 50.276}}, "truncated_count": 0}, "processing_total": {"avg_ms": 57.552, "count": 68, "max_ms": 81.951, "p95_ms": 62.973}, "read_interval": {"avg_ms": 447.123, "count": 67, "max_ms": 14976.012, "p95_ms": 86.582}, "triton_wait": {"avg_ms": 47.396, "count": 68, "max_ms": 71.496, "p95_ms": 52.435}, "visualize": {"avg_ms": 3.468, "count": 68, "max_ms": 3.803, "p95_ms": 3.697}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-01T11:32:50+09:00", "state": "running", "task_id": "4b312218-a679-4a4b-8c14-d4ec265f4513", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_113250_87d67a4e.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 49}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4252.051, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:44:54+09:00", "last_error_summary": "[h264 @ 0x55a5f4bc5000] error while decoding MB 55 42, bytestream -13 [in#0/rtsp @ 0x55a5f4b7de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a5f4b7de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:44:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 468.586, "last_started_at": "2026-07-01T11:37:05+09:00", "pid": 1379, "process": {"exists": false, "pid": 1379}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.723, "sample_count": 2781, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5972, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:44:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "87d67a4e", "last_frame_pushed_age_sec": 0.023, "last_frame_pushed_at": "2026-07-01T11:44:54+09:00", "last_frame_read_age_sec": 0.081, "last_frame_read_at": "2026-07-01T11:44:53+09:00", "mjpeg": {"age_sec": 0.009, "exists": true, "mtime": "2026-07-01T11:44:54.048234+09:00", "mtime_epoch": 1782873894.0482335, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:44:54+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1131.152, "name": "celery", "pid": 267, "rss_mb": 868.004, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.268, "pushed": 2.268, "read": 2.235}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.981, "count": 68, "max_ms": 2.463, "p95_ms": 2.103}, "gstreamer_push": {"avg_ms": 1.398, "count": 68, "max_ms": 3.828, "p95_ms": 1.975}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.547, "count": 68, "max_ms": 48.761, "p95_ms": 43.335}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.351, "count": 68, "max_ms": 71.487, "p95_ms": 52.423}, "yolo11_640_fire_11class": {"avg_ms": 41.853, "count": 68, "max_ms": 47.694, "p95_ms": 45.97}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.319, "count": 68, "max_ms": 68.317, "p95_ms": 50.276}}, "truncated_count": 0}, "processing_total": {"avg_ms": 57.552, "count": 68, "max_ms": 81.951, "p95_ms": 62.973}, "read_interval": {"avg_ms": 447.123, "count": 67, "max_ms": 14976.012, "p95_ms": 86.582}, "triton_wait": {"avg_ms": 47.396, "count": 68, "max_ms": 71.496, "p95_ms": 52.435}, "visualize": {"avg_ms": 3.468, "count": 68, "max_ms": 3.803, "p95_ms": 3.697}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T11:32:50+09:00", "state": "running", "task_id": "4b312218-a679-4a4b-8c14-d4ec265f4513", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_113250_87d67a4e.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 3, "tracking_summary": 49}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4376.004, "limit_mb": 8796093022207.996}, "detail": {"pid": 2020, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:44:54+09:00", "last_error_summary": "[h264 @ 0x55a5f4bc5000] error while decoding MB 55 42, bytestream -13 [in#0/rtsp @ 0x55a5f4b7de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a5f4b7de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T11:44:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:44:56+09:00", "pid": 2020, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.84, "name": "ffmpeg", "pid": 2020, "rss_mb": 156.676, "state": "R (running)", "threads": 49}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.461, "sample_count": 2751, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5972, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:44:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "87d67a4e", "last_frame_pushed_age_sec": 4.383, "last_frame_pushed_at": "2026-07-01T11:44:54+09:00", "last_frame_read_age_sec": 4.441, "last_frame_read_at": "2026-07-01T11:44:53+09:00", "mjpeg": {"age_sec": 4.369, "exists": true, "mtime": "2026-07-01T11:44:54.048234+09:00", "mtime_epoch": 1782873894.0482335, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:44:58+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1131.152, "name": "celery", "pid": 267, "rss_mb": 868.004, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.844, "pushed": 0.844, "read": 0.844}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.945, "count": 13, "max_ms": 2.098, "p95_ms": 2.098}, "gstreamer_push": {"avg_ms": 1.347, "count": 13, "max_ms": 1.975, "p95_ms": 1.975}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.859, "count": 13, "max_ms": 37.108, "p95_ms": 37.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 46.684, "count": 13, "max_ms": 58.299, "p95_ms": 58.299}, "yolo11_640_fire_11class": {"avg_ms": 42.538, "count": 13, "max_ms": 45.97, "p95_ms": 45.97}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 44.107, "count": 13, "max_ms": 55.946, "p95_ms": 55.946}}, "truncated_count": 0}, "processing_total": {"avg_ms": 56.879, "count": 13, "max_ms": 68.982, "p95_ms": 68.982}, "read_interval": {"avg_ms": 1994.578, "count": 13, "max_ms": 14976.012, "p95_ms": 14976.012}, "triton_wait": {"avg_ms": 46.739, "count": 13, "max_ms": 58.31, "p95_ms": 58.31}, "visualize": {"avg_ms": 3.434, "count": 13, "max_ms": 3.675, "p95_ms": 3.675}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 15.395, "window_s": 30.0}, "started_at": "2026-07-01T11:32:50+09:00", "state": "running", "task_id": "4b312218-a679-4a4b-8c14-d4ec265f4513", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 24}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4322.871, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:44:30+09:00", "last_error_summary": "[h264 @ 0x555c4431d380] error while decoding MB 52 10, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:44:30+09:00", "last_error_summary": "[h264 @ 0x555c4431d380] error while decoding MB 52 10, bytestream -5", "last_exit_at": "2026-07-01T11:46:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.495, "last_started_at": "2026-07-01T11:44:33+09:00", "pid": 1913, "process": {"exists": false, "pid": 1913}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 299.311, "sample_count": 1106, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2933, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:46:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:46:34+09:00", "last_frame_read_age_sec": 0.31, "last_frame_read_at": "2026-07-01T11:46:34+09:00", "mjpeg": {"age_sec": 0.302, "exists": true, "mtime": "2026-07-01T11:46:34.649968+09:00", "mtime_epoch": 1782873994.6499684, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173379}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:46:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1402.328, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.373, "pushed": 3.373, "read": 3.34}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.97, "count": 100, "max_ms": 19.154, "p95_ms": 14.522}, "gstreamer_push": {"avg_ms": 0.85, "count": 101, "max_ms": 1.197, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.51, "count": 100, "max_ms": 49.105, "p95_ms": 37.134}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.775, "count": 100, "max_ms": 85.682, "p95_ms": 78.577}, "yolo11_640_fire_11class": {"avg_ms": 45.507, "count": 100, "max_ms": 59.411, "p95_ms": 54.837}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.588, "count": 100, "max_ms": 82.794, "p95_ms": 80.609}}, "truncated_count": 0}, "processing_total": {"avg_ms": 289.867, "count": 101, "max_ms": 344.352, "p95_ms": 308.082}, "read_interval": {"avg_ms": 299.334, "count": 100, "max_ms": 353.978, "p95_ms": 318.677}, "triton_wait": {"avg_ms": 67.463, "count": 100, "max_ms": 85.695, "p95_ms": 81.338}, "visualize": {"avg_ms": 13.555, "count": 100, "max_ms": 17.39, "p95_ms": 14.743}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 24}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4322.863, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:44:30+09:00", "last_error_summary": "[h264 @ 0x555c4431d380] error while decoding MB 52 10, bytestream -5", "last_exit_at": "2026-07-01T11:46:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.498, "last_started_at": "2026-07-01T11:44:33+09:00", "pid": 1913, "process": {"exists": false, "pid": 1913}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 299.311, "sample_count": 1106, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2933, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:46:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T11:46:34+09:00", "last_frame_read_age_sec": 0.313, "last_frame_read_at": "2026-07-01T11:46:34+09:00", "mjpeg": {"age_sec": 0.305, "exists": true, "mtime": "2026-07-01T11:46:34.649968+09:00", "mtime_epoch": 1782873994.6499684, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173379}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:46:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1402.328, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.373, "pushed": 3.373, "read": 3.339}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.97, "count": 100, "max_ms": 19.154, "p95_ms": 14.522}, "gstreamer_push": {"avg_ms": 0.85, "count": 101, "max_ms": 1.197, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.51, "count": 100, "max_ms": 49.105, "p95_ms": 37.134}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.775, "count": 100, "max_ms": 85.682, "p95_ms": 78.577}, "yolo11_640_fire_11class": {"avg_ms": 45.507, "count": 100, "max_ms": 59.411, "p95_ms": 54.837}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.588, "count": 100, "max_ms": 82.794, "p95_ms": 80.609}}, "truncated_count": 0}, "processing_total": {"avg_ms": 289.867, "count": 101, "max_ms": 344.352, "p95_ms": 308.082}, "read_interval": {"avg_ms": 299.334, "count": 100, "max_ms": 353.978, "p95_ms": 318.677}, "triton_wait": {"avg_ms": 67.463, "count": 100, "max_ms": 85.695, "p95_ms": 81.338}, "visualize": {"avg_ms": 13.555, "count": 100, "max_ms": 17.39, "p95_ms": 14.743}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 24}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4428.203, "limit_mb": 8796093022207.996}, "detail": {"pid": 2172, "restart_count": 7, "start_count": 8}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:44:30+09:00", "last_error_summary": "[h264 @ 0x555c4431d380] error while decoding MB 52 10, bytestream -5", "last_exit_at": "2026-07-01T11:46:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:46:37+09:00", "pid": 2172, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.633, "name": "ffmpeg", "pid": 2172, "rss_mb": 136.633, "state": "S (sleeping)", "threads": 106}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 300.168, "sample_count": 1088, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2933, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:46:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 4.257, "last_frame_pushed_at": "2026-07-01T11:46:34+09:00", "last_frame_read_age_sec": 4.564, "last_frame_read_at": "2026-07-01T11:46:34+09:00", "mjpeg": {"age_sec": 4.241, "exists": true, "mtime": "2026-07-01T11:46:34.965974+09:00", "mtime_epoch": 1782873994.9659739, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173629}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:46:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1402.328, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.894, "pushed": 2.894, "read": 2.86}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.931, "count": 86, "max_ms": 19.154, "p95_ms": 14.522}, "gstreamer_push": {"avg_ms": 0.849, "count": 86, "max_ms": 1.197, "p95_ms": 0.882}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.34, "count": 86, "max_ms": 43.602, "p95_ms": 37.134}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.914, "count": 86, "max_ms": 85.682, "p95_ms": 78.662}, "yolo11_640_fire_11class": {"avg_ms": 45.519, "count": 86, "max_ms": 59.411, "p95_ms": 55.616}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.554, "count": 86, "max_ms": 82.794, "p95_ms": 80.881}}, "truncated_count": 0}, "processing_total": {"avg_ms": 289.858, "count": 86, "max_ms": 344.352, "p95_ms": 310.909}, "read_interval": {"avg_ms": 299.342, "count": 85, "max_ms": 353.978, "p95_ms": 321.428}, "triton_wait": {"avg_ms": 67.579, "count": 86, "max_ms": 85.695, "p95_ms": 81.652}, "visualize": {"avg_ms": 13.534, "count": 86, "max_ms": 17.39, "p95_ms": 14.808}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.72, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 27}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4324.355, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563a9ee5b480] error while decoding MB 13 28, bytestream -26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:48:26+09:00", "last_error_summary": "[h264 @ 0x563a9ee5b480] error while decoding MB 13 28, bytestream -26", "last_exit_at": "2026-07-01T11:46:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.097, "last_started_at": "2026-07-01T11:46:37+09:00", "pid": 2172, "process": {"exists": false, "pid": 2172}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 300.39, "sample_count": 1076, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3352, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:48:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T11:48:26+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T11:48:26+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T11:48:26.263885+09:00", "mtime_epoch": 1782874106.2638845, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168791}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:48:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1402.328, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.279, "pushed": 4.279, "read": 4.279}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.704, "count": 127, "max_ms": 14.642, "p95_ms": 10.857}, "gstreamer_push": {"avg_ms": 0.852, "count": 128, "max_ms": 0.98, "p95_ms": 0.893}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.745, "count": 127, "max_ms": 49.413, "p95_ms": 42.042}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.91, "count": 127, "max_ms": 84.763, "p95_ms": 80.224}, "yolo11_640_fire_11class": {"avg_ms": 47.151, "count": 127, "max_ms": 61.402, "p95_ms": 57.508}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.169, "count": 127, "max_ms": 83.916, "p95_ms": 77.805}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.794, "count": 128, "max_ms": 269.526, "p95_ms": 257.602}, "read_interval": {"avg_ms": 235.183, "count": 128, "max_ms": 279.876, "p95_ms": 266.616}, "triton_wait": {"avg_ms": 66.018, "count": 127, "max_ms": 84.775, "p95_ms": 80.689}, "visualize": {"avg_ms": 10.714, "count": 127, "max_ms": 12.636, "p95_ms": 12.161}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 27}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4323.48, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:48:26+09:00", "last_error_summary": "[h264 @ 0x563a9ee5b480] error while decoding MB 13 28, bytestream -26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:48:26+09:00", "last_error_summary": "[h264 @ 0x563a9ee5b480] error while decoding MB 13 28, bytestream -26", "last_exit_at": "2026-07-01T11:48:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.285, "last_started_at": "2026-07-01T11:46:37+09:00", "pid": 2172, "process": {"exists": false, "pid": 2172}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 300.39, "sample_count": 1076, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3353, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:48:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T11:48:26+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T11:48:26+09:00", "mjpeg": {"age_sec": 0.227, "exists": true, "mtime": "2026-07-01T11:48:26.263885+09:00", "mtime_epoch": 1782874106.2638845, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168791}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:48:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1402.328, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.286, "pushed": 4.286, "read": 4.252}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.712, "count": 127, "max_ms": 14.642, "p95_ms": 10.865}, "gstreamer_push": {"avg_ms": 0.852, "count": 128, "max_ms": 0.98, "p95_ms": 0.893}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.75, "count": 127, "max_ms": 49.413, "p95_ms": 42.042}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.907, "count": 127, "max_ms": 84.763, "p95_ms": 80.224}, "yolo11_640_fire_11class": {"avg_ms": 47.144, "count": 127, "max_ms": 61.402, "p95_ms": 57.508}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.183, "count": 127, "max_ms": 83.916, "p95_ms": 77.805}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.73, "count": 128, "max_ms": 269.526, "p95_ms": 257.602}, "read_interval": {"avg_ms": 235.068, "count": 127, "max_ms": 279.876, "p95_ms": 266.616}, "triton_wait": {"avg_ms": 66.015, "count": 127, "max_ms": 84.775, "p95_ms": 80.689}, "visualize": {"avg_ms": 10.721, "count": 127, "max_ms": 12.636, "p95_ms": 12.161}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 27}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4323.473, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:48:26+09:00", "last_error_summary": "[h264 @ 0x563a9ee5b480] error while decoding MB 13 28, bytestream -26", "last_exit_at": "2026-07-01T11:48:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.288, "last_started_at": "2026-07-01T11:46:37+09:00", "pid": 2172, "process": {"exists": false, "pid": 2172}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 300.39, "sample_count": 1076, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3353, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:48:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T11:48:26+09:00", "last_frame_read_age_sec": 0.241, "last_frame_read_at": "2026-07-01T11:48:26+09:00", "mjpeg": {"age_sec": 0.23, "exists": true, "mtime": "2026-07-01T11:48:26.263885+09:00", "mtime_epoch": 1782874106.2638845, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168791}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:48:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1402.328, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.285, "pushed": 4.285, "read": 4.252}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.712, "count": 127, "max_ms": 14.642, "p95_ms": 10.865}, "gstreamer_push": {"avg_ms": 0.852, "count": 128, "max_ms": 0.98, "p95_ms": 0.893}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.75, "count": 127, "max_ms": 49.413, "p95_ms": 42.042}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.907, "count": 127, "max_ms": 84.763, "p95_ms": 80.224}, "yolo11_640_fire_11class": {"avg_ms": 47.144, "count": 127, "max_ms": 61.402, "p95_ms": 57.508}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.183, "count": 127, "max_ms": 83.916, "p95_ms": 77.805}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.73, "count": 128, "max_ms": 269.526, "p95_ms": 257.602}, "read_interval": {"avg_ms": 235.068, "count": 127, "max_ms": 279.876, "p95_ms": 266.616}, "triton_wait": {"avg_ms": 66.015, "count": 127, "max_ms": 84.775, "p95_ms": 80.689}, "visualize": {"avg_ms": 10.721, "count": 127, "max_ms": 12.636, "p95_ms": 12.161}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 27}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4428.863, "limit_mb": 8796093022207.996}, "detail": {"pid": 2279, "restart_count": 8, "start_count": 9}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:48:26+09:00", "last_error_summary": "[h264 @ 0x563a9ee5b480] error while decoding MB 13 28, bytestream -26", "last_exit_at": "2026-07-01T11:48:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:48:29+09:00", "pid": 2279, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.113, "name": "ffmpeg", "pid": 2279, "rss_mb": 136.113, "state": "S (sleeping)", "threads": 106}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 300.533, "sample_count": 1056, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3353, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:48:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 4.521, "last_frame_pushed_at": "2026-07-01T11:48:26+09:00", "last_frame_read_age_sec": 4.754, "last_frame_read_at": "2026-07-01T11:48:26+09:00", "mjpeg": {"age_sec": 4.503, "exists": true, "mtime": "2026-07-01T11:48:26.503889+09:00", "mtime_epoch": 1782874106.5038886, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166704}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:48:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1402.328, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.656, "pushed": 3.656, "read": 3.623}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.656, "count": 109, "max_ms": 14.642, "p95_ms": 10.857}, "gstreamer_push": {"avg_ms": 0.85, "count": 109, "max_ms": 0.98, "p95_ms": 0.896}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.837, "count": 109, "max_ms": 49.413, "p95_ms": 44.883}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.54, "count": 109, "max_ms": 84.763, "p95_ms": 79.072}, "yolo11_640_fire_11class": {"avg_ms": 47.116, "count": 109, "max_ms": 61.402, "p95_ms": 56.571}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.05, "count": 109, "max_ms": 83.916, "p95_ms": 77.461}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.773, "count": 109, "max_ms": 269.526, "p95_ms": 258.091}, "read_interval": {"avg_ms": 234.088, "count": 108, "max_ms": 279.876, "p95_ms": 267.395}, "triton_wait": {"avg_ms": 65.722, "count": 109, "max_ms": 84.775, "p95_ms": 80.689}, "visualize": {"avg_ms": 10.707, "count": 109, "max_ms": 12.636, "p95_ms": 12.177}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.811, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 31}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4478.965, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5601e4796380] error while decoding MB 67 37, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:50:38+09:00", "last_error_summary": "[h264 @ 0x5601e4796380] error while decoding MB 67 37, bytestream -5", "last_exit_at": "2026-07-01T11:48:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.723, "last_started_at": "2026-07-01T11:48:29+09:00", "pid": 2279, "process": {"exists": false, "pid": 2279}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 380.88, "sample_count": 1054, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3799, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:50:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T11:50:38+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T11:50:38+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T11:50:38.686148+09:00", "mtime_epoch": 1782874238.6861482, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176570}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:50:38+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1403.094, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 20.389, "recorders": [{"age_sec": 20.389, "frame_count": 95, "is_recording": true, "is_stopping": false, "pid": 2386, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.688, "name": "ffmpeg", "pid": 2386, "rss_mb": 186.688, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 5, "returncode": null, "running": true, "started_at": "2026-07-01T11:50:18+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_115018_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [2386], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 70, "pushed_frames": 70, "read_frames": 70, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.356, "pushed": 2.356, "read": 2.356}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.675, "count": 70, "max_ms": 28.255, "p95_ms": 25.486}, "gstreamer_push": {"avg_ms": 0.858, "count": 70, "max_ms": 0.91, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.421, "count": 70, "max_ms": 52.936, "p95_ms": 45.25}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.315, "count": 70, "max_ms": 90.158, "p95_ms": 85.487}, "yolo11_640_fire_11class": {"avg_ms": 45.362, "count": 70, "max_ms": 65.093, "p95_ms": 56.611}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.788, "count": 70, "max_ms": 91.787, "p95_ms": 84.583}}, "truncated_count": 0}, "processing_total": {"avg_ms": 419.218, "count": 70, "max_ms": 1052.857, "p95_ms": 462.719}, "read_interval": {"avg_ms": 428.86, "count": 70, "max_ms": 1063.041, "p95_ms": 471.633}, "triton_wait": {"avg_ms": 71.84, "count": 70, "max_ms": 91.799, "p95_ms": 86.504}, "visualize": {"avg_ms": 19.491, "count": 70, "max_ms": 25.515, "p95_ms": 24.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.712, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 31}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4477.465, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:50:38+09:00", "last_error_summary": "[h264 @ 0x5601e4796380] error while decoding MB 67 37, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:50:38+09:00", "last_error_summary": "[h264 @ 0x5601e4796380] error while decoding MB 67 37, bytestream -5", "last_exit_at": "2026-07-01T11:50:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.157, "last_started_at": "2026-07-01T11:48:29+09:00", "pid": 2279, "process": {"exists": false, "pid": 2279}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 382.452, "sample_count": 1054, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3800, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:50:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T11:50:39+09:00", "last_frame_read_age_sec": 0.487, "last_frame_read_at": "2026-07-01T11:50:38+09:00", "mjpeg": {"age_sec": 0.477, "exists": true, "mtime": "2026-07-01T11:50:38.686148+09:00", "mtime_epoch": 1782874238.6861482, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176570}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:50:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1403.094, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 20.823, "recorders": [{"age_sec": 20.823, "frame_count": 96, "is_recording": true, "is_stopping": false, "pid": 2386, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.688, "name": "ffmpeg", "pid": 2386, "rss_mb": 186.688, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 4, "returncode": null, "running": true, "started_at": "2026-07-01T11:50:18+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_115018_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [2386], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 70, "pushed_frames": 70, "read_frames": 69, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.351, "pushed": 2.351, "read": 2.317}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.83, "count": 69, "max_ms": 28.255, "p95_ms": 25.486}, "gstreamer_push": {"avg_ms": 0.858, "count": 70, "max_ms": 0.91, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.784, "count": 69, "max_ms": 52.936, "p95_ms": 47.019}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.424, "count": 69, "max_ms": 90.158, "p95_ms": 85.487}, "yolo11_640_fire_11class": {"avg_ms": 45.582, "count": 69, "max_ms": 65.093, "p95_ms": 56.611}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.813, "count": 69, "max_ms": 91.787, "p95_ms": 84.583}}, "truncated_count": 0}, "processing_total": {"avg_ms": 420.943, "count": 70, "max_ms": 1052.857, "p95_ms": 476.026}, "read_interval": {"avg_ms": 429.726, "count": 69, "max_ms": 1063.041, "p95_ms": 471.633}, "triton_wait": {"avg_ms": 71.941, "count": 69, "max_ms": 91.799, "p95_ms": 86.504}, "visualize": {"avg_ms": 19.653, "count": 69, "max_ms": 25.515, "p95_ms": 24.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.774, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 31}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4477.484, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:50:38+09:00", "last_error_summary": "[h264 @ 0x5601e4796380] error while decoding MB 67 37, bytestream -5", "last_exit_at": "2026-07-01T11:50:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.161, "last_started_at": "2026-07-01T11:48:29+09:00", "pid": 2279, "process": {"exists": false, "pid": 2279}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 382.452, "sample_count": 1054, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3800, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:50:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T11:50:39+09:00", "last_frame_read_age_sec": 0.49, "last_frame_read_at": "2026-07-01T11:50:38+09:00", "mjpeg": {"age_sec": 0.48, "exists": true, "mtime": "2026-07-01T11:50:38.686148+09:00", "mtime_epoch": 1782874238.6861482, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176570}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:50:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1403.094, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 20.826, "recorders": [{"age_sec": 20.826, "frame_count": 96, "is_recording": true, "is_stopping": false, "pid": 2386, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.688, "name": "ffmpeg", "pid": 2386, "rss_mb": 186.688, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 4, "returncode": null, "running": true, "started_at": "2026-07-01T11:50:18+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_115018_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [2386], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 70, "pushed_frames": 70, "read_frames": 69, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.351, "pushed": 2.351, "read": 2.317}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.83, "count": 69, "max_ms": 28.255, "p95_ms": 25.486}, "gstreamer_push": {"avg_ms": 0.858, "count": 70, "max_ms": 0.91, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.784, "count": 69, "max_ms": 52.936, "p95_ms": 47.019}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.424, "count": 69, "max_ms": 90.158, "p95_ms": 85.487}, "yolo11_640_fire_11class": {"avg_ms": 45.582, "count": 69, "max_ms": 65.093, "p95_ms": 56.611}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.813, "count": 69, "max_ms": 91.787, "p95_ms": 84.583}}, "truncated_count": 0}, "processing_total": {"avg_ms": 420.943, "count": 70, "max_ms": 1052.857, "p95_ms": 476.026}, "read_interval": {"avg_ms": 429.726, "count": 69, "max_ms": 1063.041, "p95_ms": 471.633}, "triton_wait": {"avg_ms": 71.941, "count": 69, "max_ms": 91.799, "p95_ms": 86.504}, "visualize": {"avg_ms": 19.653, "count": 69, "max_ms": 25.515, "p95_ms": 24.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.777, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_113252_f923dc33.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 31}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4580.898, "limit_mb": 8796093022207.996}, "detail": {"pid": 2446, "restart_count": 9, "start_count": 10}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:50:38+09:00", "last_error_summary": "[h264 @ 0x5601e4796380] error while decoding MB 67 37, bytestream -5", "last_exit_at": "2026-07-01T11:50:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:50:41+09:00", "pid": 2446, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.973, "name": "ffmpeg", "pid": 2446, "rss_mb": 133.973, "state": "S (sleeping)", "threads": 106}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 385.288, "sample_count": 1038, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3800, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:50:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f923dc33", "last_frame_pushed_age_sec": 4.517, "last_frame_pushed_at": "2026-07-01T11:50:39+09:00", "last_frame_read_age_sec": 4.999, "last_frame_read_at": "2026-07-01T11:50:38+09:00", "mjpeg": {"age_sec": 4.497, "exists": true, "mtime": "2026-07-01T11:50:39.178157+09:00", "mtime_epoch": 1782874239.1781566, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175596}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:50:43+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1403.094, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 25.336, "recorders": [{"age_sec": 25.336, "frame_count": 96, "is_recording": true, "is_stopping": false, "pid": 2386, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.688, "name": "ffmpeg", "pid": 2386, "rss_mb": 186.688, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 4, "returncode": null, "running": true, "started_at": "2026-07-01T11:50:18+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_115018_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [2386], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.944, "pushed": 1.944, "read": 1.91}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.584, "count": 57, "max_ms": 28.255, "p95_ms": 25.648}, "gstreamer_push": {"avg_ms": 0.857, "count": 58, "max_ms": 0.91, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.385, "count": 57, "max_ms": 52.936, "p95_ms": 47.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.486, "count": 57, "max_ms": 90.158, "p95_ms": 85.746}, "yolo11_640_fire_11class": {"avg_ms": 46.587, "count": 57, "max_ms": 65.093, "p95_ms": 59.436}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.944, "count": 57, "max_ms": 91.787, "p95_ms": 88.719}}, "truncated_count": 0}, "processing_total": {"avg_ms": 433.917, "count": 58, "max_ms": 1052.857, "p95_ms": 480.114}, "read_interval": {"avg_ms": 442.769, "count": 57, "max_ms": 1063.041, "p95_ms": 485.299}, "triton_wait": {"avg_ms": 72.038, "count": 57, "max_ms": 91.799, "p95_ms": 89.394}, "visualize": {"avg_ms": 20.476, "count": 57, "max_ms": 25.515, "p95_ms": 24.971}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.842, "window_s": 30.0}, "started_at": "2026-07-01T11:32:52+09:00", "state": "running", "task_id": "69919e14-028d-4319-8172-0dfba64e8cc5", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_115104_ab775f64.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4140.562, "limit_mb": 8796093022207.996}, "detail": {"pid": 2554, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:51:04+09:00", "pid": 2554, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.539, "name": "ffmpeg", "pid": 2554, "rss_mb": 147.539, "state": "R (running)", "threads": 49}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.937}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "ab775f64", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 2.615, "exists": true, "mtime": "2026-07-01T11:51:03.726575+09:00", "mtime_epoch": 1782874263.7265751, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230626}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T11:51:06+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1131.152, "name": "celery", "pid": 267, "rss_mb": 863.617, "state": "R (running)", "threads": 120}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:51:04+09:00", "state": "running", "task_id": "64e6ede9-0997-463c-b342-3af5d237a507", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4472.0, "limit_mb": 8796093022207.996}, "detail": {"pid": 2557, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:51:05+09:00", "pid": 2557, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.277, "name": "ffmpeg", "pid": 2557, "rss_mb": 139.277, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.939}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 2.663, "exists": true, "mtime": "2026-07-01T11:51:05.146599+09:00", "mtime_epoch": 1782874265.1465993, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172184}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:51:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1516.941, "name": "celery", "pid": 237, "rss_mb": 1402.035, "state": "R (running)", "threads": 120}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5111.559, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c73cb75100] error while decoding MB 9 7, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:53:06+09:00", "last_error_summary": "[h264 @ 0x55c73cb75100] error while decoding MB 9 7, bytestream -12", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 120.423, "last_started_at": "2026-07-01T11:51:05+09:00", "pid": 2557, "process": {"exists": false, "pid": 2557}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 364.895, "sample_count": 392, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 179.516}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:53:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T11:53:06+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T11:53:06+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T11:53:06.188660+09:00", "mtime_epoch": 1782874386.18866, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:53:06+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1906.77, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.41, "pushed": 3.41, "read": 3.41}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.754, "count": 101, "max_ms": 17.607, "p95_ms": 16.189}, "gstreamer_push": {"avg_ms": 0.838, "count": 102, "max_ms": 0.992, "p95_ms": 0.875}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.973, "count": 101, "max_ms": 49.338, "p95_ms": 43.64}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.721, "count": 101, "max_ms": 87.528, "p95_ms": 82.695}, "yolo11_640_fire_11class": {"avg_ms": 47.751, "count": 101, "max_ms": 65.074, "p95_ms": 60.313}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.269, "count": 101, "max_ms": 87.351, "p95_ms": 82.637}}, "truncated_count": 0}, "processing_total": {"avg_ms": 286.193, "count": 102, "max_ms": 411.897, "p95_ms": 329.63}, "read_interval": {"avg_ms": 295.205, "count": 102, "max_ms": 420.375, "p95_ms": 338.531}, "triton_wait": {"avg_ms": 70.441, "count": 101, "max_ms": 87.54, "p95_ms": 83.213}, "visualize": {"avg_ms": 14.247, "count": 101, "max_ms": 20.688, "p95_ms": 19.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5110.719, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:53:06+09:00", "last_error_summary": "[h264 @ 0x55c73cb75100] error while decoding MB 9 7, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:53:06+09:00", "last_error_summary": "[h264 @ 0x55c73cb75100] error while decoding MB 9 7, bytestream -12", "last_exit_at": "2026-07-01T11:53:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.667, "last_started_at": "2026-07-01T11:51:05+09:00", "pid": 2557, "process": {"exists": false, "pid": 2557}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 364.895, "sample_count": 393, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 179.273}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 393, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:53:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-01T11:53:06+09:00", "last_frame_read_age_sec": 0.293, "last_frame_read_at": "2026-07-01T11:53:06+09:00", "mjpeg": {"age_sec": 0.286, "exists": true, "mtime": "2026-07-01T11:53:06.188660+09:00", "mtime_epoch": 1782874386.18866, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:53:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1906.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.411, "pushed": 3.411, "read": 3.378}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.77, "count": 101, "max_ms": 17.607, "p95_ms": 16.189}, "gstreamer_push": {"avg_ms": 0.837, "count": 102, "max_ms": 0.992, "p95_ms": 0.87}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.962, "count": 101, "max_ms": 49.338, "p95_ms": 43.64}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.781, "count": 101, "max_ms": 87.528, "p95_ms": 82.695}, "yolo11_640_fire_11class": {"avg_ms": 47.726, "count": 101, "max_ms": 65.074, "p95_ms": 60.313}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.31, "count": 101, "max_ms": 87.351, "p95_ms": 82.637}}, "truncated_count": 0}, "processing_total": {"avg_ms": 286.612, "count": 102, "max_ms": 411.897, "p95_ms": 329.63}, "read_interval": {"avg_ms": 295.602, "count": 101, "max_ms": 420.375, "p95_ms": 338.531}, "triton_wait": {"avg_ms": 70.501, "count": 101, "max_ms": 87.54, "p95_ms": 83.213}, "visualize": {"avg_ms": 14.27, "count": 101, "max_ms": 20.688, "p95_ms": 19.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5110.711, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:53:06+09:00", "last_error_summary": "[h264 @ 0x55c73cb75100] error while decoding MB 9 7, bytestream -12", "last_exit_at": "2026-07-01T11:53:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.67, "last_started_at": "2026-07-01T11:51:05+09:00", "pid": 2557, "process": {"exists": false, "pid": 2557}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 364.895, "sample_count": 393, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 179.27}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 393, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:53:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T11:53:06+09:00", "last_frame_read_age_sec": 0.296, "last_frame_read_at": "2026-07-01T11:53:06+09:00", "mjpeg": {"age_sec": 0.289, "exists": true, "mtime": "2026-07-01T11:53:06.188660+09:00", "mtime_epoch": 1782874386.18866, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:53:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1906.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.411, "pushed": 3.411, "read": 3.377}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.77, "count": 101, "max_ms": 17.607, "p95_ms": 16.189}, "gstreamer_push": {"avg_ms": 0.837, "count": 102, "max_ms": 0.992, "p95_ms": 0.87}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.962, "count": 101, "max_ms": 49.338, "p95_ms": 43.64}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.781, "count": 101, "max_ms": 87.528, "p95_ms": 82.695}, "yolo11_640_fire_11class": {"avg_ms": 47.726, "count": 101, "max_ms": 65.074, "p95_ms": 60.313}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.31, "count": 101, "max_ms": 87.351, "p95_ms": 82.637}}, "truncated_count": 0}, "processing_total": {"avg_ms": 286.612, "count": 102, "max_ms": 411.897, "p95_ms": 329.63}, "read_interval": {"avg_ms": 295.602, "count": 101, "max_ms": 420.375, "p95_ms": 338.531}, "triton_wait": {"avg_ms": 70.501, "count": 101, "max_ms": 87.54, "p95_ms": 83.213}, "visualize": {"avg_ms": 14.27, "count": 101, "max_ms": 20.688, "p95_ms": 19.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5221.332, "limit_mb": 8796093022207.996}, "detail": {"pid": 2877, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:53:06+09:00", "last_error_summary": "[h264 @ 0x55c73cb75100] error while decoding MB 9 7, bytestream -12", "last_exit_at": "2026-07-01T11:53:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:53:08+09:00", "pid": 2877, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.266, "name": "ffmpeg", "pid": 2877, "rss_mb": 139.266, "state": "S (sleeping)", "threads": 106}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 364.895, "sample_count": 393, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 174.796}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 393, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:53:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.479, "last_frame_pushed_at": "2026-07-01T11:53:06+09:00", "last_frame_read_age_sec": 4.77, "last_frame_read_at": "2026-07-01T11:53:06+09:00", "mjpeg": {"age_sec": 4.463, "exists": true, "mtime": "2026-07-01T11:53:06.488665+09:00", "mtime_epoch": 1782874386.4886649, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173468}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:53:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1906.773, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 83, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.822, "pushed": 2.822, "read": 2.788}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.307, "count": 84, "max_ms": 17.607, "p95_ms": 16.41}, "gstreamer_push": {"avg_ms": 0.838, "count": 84, "max_ms": 0.992, "p95_ms": 0.875}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.077, "count": 84, "max_ms": 49.338, "p95_ms": 43.64}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.38, "count": 84, "max_ms": 87.528, "p95_ms": 83.075}, "yolo11_640_fire_11class": {"avg_ms": 47.539, "count": 84, "max_ms": 65.074, "p95_ms": 58.654}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.65, "count": 84, "max_ms": 87.351, "p95_ms": 83.509}}, "truncated_count": 0}, "processing_total": {"avg_ms": 294.999, "count": 84, "max_ms": 411.897, "p95_ms": 329.91}, "read_interval": {"avg_ms": 304.06, "count": 83, "max_ms": 420.375, "p95_ms": 338.61}, "triton_wait": {"avg_ms": 71.052, "count": 84, "max_ms": 87.54, "p95_ms": 83.521}, "visualize": {"avg_ms": 14.771, "count": 84, "max_ms": 20.688, "p95_ms": 19.711}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.766, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5115.375, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560ab4b51200] error while decoding MB 49 20, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:54:59+09:00", "last_error_summary": "[h264 @ 0x560ab4b51200] error while decoding MB 49 20, bytestream -11", "last_exit_at": "2026-07-01T11:53:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.726, "last_started_at": "2026-07-01T11:53:08+09:00", "pid": 2877, "process": {"exists": false, "pid": 2877}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 347.033, "sample_count": 816, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 66.07}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 816, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:54:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-01T11:54:59+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T11:54:59+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-01T11:54:59.638586+09:00", "mtime_epoch": 1782874499.6385863, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174687}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:54:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1906.773, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.914, "pushed": 3.914, "read": 3.914}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.13, "count": 116, "max_ms": 13.165, "p95_ms": 11.192}, "gstreamer_push": {"avg_ms": 0.862, "count": 117, "max_ms": 1.111, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.153, "count": 116, "max_ms": 51.285, "p95_ms": 45.805}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.851, "count": 116, "max_ms": 84.077, "p95_ms": 79.257}, "yolo11_640_fire_11class": {"avg_ms": 48.739, "count": 116, "max_ms": 64.777, "p95_ms": 59.844}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.142, "count": 116, "max_ms": 88.377, "p95_ms": 79.571}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.77, "count": 117, "max_ms": 282.24, "p95_ms": 265.594}, "read_interval": {"avg_ms": 257.188, "count": 117, "max_ms": 291.1, "p95_ms": 274.579}, "triton_wait": {"avg_ms": 67.67, "count": 116, "max_ms": 88.389, "p95_ms": 80.933}, "visualize": {"avg_ms": 11.681, "count": 116, "max_ms": 16.773, "p95_ms": 12.75}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5114.359, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:54:59+09:00", "last_error_summary": "[h264 @ 0x560ab4b51200] error while decoding MB 49 20, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:54:59+09:00", "last_error_summary": "[h264 @ 0x560ab4b51200] error while decoding MB 49 20, bytestream -11", "last_exit_at": "2026-07-01T11:54:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.941, "last_started_at": "2026-07-01T11:53:08+09:00", "pid": 2877, "process": {"exists": false, "pid": 2877}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 347.033, "sample_count": 817, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 65.856}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 817, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:54:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:54:59+09:00", "last_frame_read_age_sec": 0.264, "last_frame_read_at": "2026-07-01T11:54:59+09:00", "mjpeg": {"age_sec": 0.253, "exists": true, "mtime": "2026-07-01T11:54:59.638586+09:00", "mtime_epoch": 1782874499.6385863, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174687}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:54:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1906.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.921, "pushed": 3.921, "read": 3.887}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.135, "count": 116, "max_ms": 13.165, "p95_ms": 11.192}, "gstreamer_push": {"avg_ms": 0.862, "count": 117, "max_ms": 1.111, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.154, "count": 116, "max_ms": 51.285, "p95_ms": 45.805}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.933, "count": 116, "max_ms": 84.077, "p95_ms": 79.257}, "yolo11_640_fire_11class": {"avg_ms": 48.812, "count": 116, "max_ms": 64.777, "p95_ms": 59.844}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.16, "count": 116, "max_ms": 88.377, "p95_ms": 79.571}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.875, "count": 117, "max_ms": 282.24, "p95_ms": 265.594}, "read_interval": {"avg_ms": 257.191, "count": 116, "max_ms": 291.1, "p95_ms": 274.579}, "triton_wait": {"avg_ms": 67.687, "count": 116, "max_ms": 88.389, "p95_ms": 80.933}, "visualize": {"avg_ms": 11.687, "count": 116, "max_ms": 16.773, "p95_ms": 12.75}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.841, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5114.352, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:54:59+09:00", "last_error_summary": "[h264 @ 0x560ab4b51200] error while decoding MB 49 20, bytestream -11", "last_exit_at": "2026-07-01T11:54:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.944, "last_started_at": "2026-07-01T11:53:08+09:00", "pid": 2877, "process": {"exists": false, "pid": 2877}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 347.033, "sample_count": 817, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 65.854}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 817, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:54:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T11:54:59+09:00", "last_frame_read_age_sec": 0.267, "last_frame_read_at": "2026-07-01T11:54:59+09:00", "mjpeg": {"age_sec": 0.256, "exists": true, "mtime": "2026-07-01T11:54:59.638586+09:00", "mtime_epoch": 1782874499.6385863, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174687}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:54:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1906.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.92, "pushed": 3.92, "read": 3.887}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.135, "count": 116, "max_ms": 13.165, "p95_ms": 11.192}, "gstreamer_push": {"avg_ms": 0.862, "count": 117, "max_ms": 1.111, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.154, "count": 116, "max_ms": 51.285, "p95_ms": 45.805}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.933, "count": 116, "max_ms": 84.077, "p95_ms": 79.257}, "yolo11_640_fire_11class": {"avg_ms": 48.812, "count": 116, "max_ms": 64.777, "p95_ms": 59.844}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.16, "count": 116, "max_ms": 88.377, "p95_ms": 79.571}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.875, "count": 117, "max_ms": 282.24, "p95_ms": 265.594}, "read_interval": {"avg_ms": 257.191, "count": 116, "max_ms": 291.1, "p95_ms": 274.579}, "triton_wait": {"avg_ms": 67.687, "count": 116, "max_ms": 88.389, "p95_ms": 80.933}, "visualize": {"avg_ms": 11.687, "count": 116, "max_ms": 16.773, "p95_ms": 12.75}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 6}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5225.297, "limit_mb": 8796093022207.996}, "detail": {"pid": 2984, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:54:59+09:00", "last_error_summary": "[h264 @ 0x560ab4b51200] error while decoding MB 49 20, bytestream -11", "last_exit_at": "2026-07-01T11:55:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:55:02+09:00", "pid": 2984, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.832, "name": "ffmpeg", "pid": 2984, "rss_mb": 141.832, "state": "S (sleeping)", "threads": 106}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 347.033, "sample_count": 817, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 61.362}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 817, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:54:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.496, "last_frame_pushed_at": "2026-07-01T11:54:59+09:00", "last_frame_read_age_sec": 4.757, "last_frame_read_at": "2026-07-01T11:54:59+09:00", "mjpeg": {"age_sec": 4.479, "exists": true, "mtime": "2026-07-01T11:54:59.906591+09:00", "mtime_epoch": 1782874499.906591, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:55:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1906.773, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.345, "pushed": 3.345, "read": 3.311}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.069, "count": 99, "max_ms": 13.165, "p95_ms": 11.103}, "gstreamer_push": {"avg_ms": 0.86, "count": 100, "max_ms": 1.111, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.213, "count": 99, "max_ms": 51.285, "p95_ms": 45.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.746, "count": 99, "max_ms": 84.077, "p95_ms": 79.491}, "yolo11_640_fire_11class": {"avg_ms": 48.486, "count": 99, "max_ms": 64.777, "p95_ms": 60.405}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.83, "count": 99, "max_ms": 88.377, "p95_ms": 79.571}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.152, "count": 100, "max_ms": 282.24, "p95_ms": 265.594}, "read_interval": {"avg_ms": 256.446, "count": 99, "max_ms": 291.1, "p95_ms": 274.993}, "triton_wait": {"avg_ms": 67.46, "count": 99, "max_ms": 88.389, "p95_ms": 81.05}, "visualize": {"avg_ms": 11.72, "count": 99, "max_ms": 16.773, "p95_ms": 12.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 10}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5115.625, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562aebd74a00] error while decoding MB 74 7, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:56:53+09:00", "last_error_summary": "[h264 @ 0x562aebd74a00] error while decoding MB 74 7, bytestream -5", "last_exit_at": "2026-07-01T11:55:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.997, "last_started_at": "2026-07-01T11:55:02+09:00", "pid": 2984, "process": {"exists": false, "pid": 2984}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 305.396, "sample_count": 1095, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1234, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:56:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T11:56:53+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T11:56:53+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T11:56:53.336514+09:00", "mtime_epoch": 1782874613.3365135, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177134}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:56:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.027, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.818, "pushed": 3.818, "read": 3.818}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.36, "count": 113, "max_ms": 17.372, "p95_ms": 11.22}, "gstreamer_push": {"avg_ms": 0.851, "count": 114, "max_ms": 0.936, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.079, "count": 113, "max_ms": 48.227, "p95_ms": 41.393}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.336, "count": 113, "max_ms": 87.579, "p95_ms": 83.899}, "yolo11_640_fire_11class": {"avg_ms": 48.22, "count": 113, "max_ms": 61.184, "p95_ms": 58.867}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.755, "count": 113, "max_ms": 84.776, "p95_ms": 80.541}}, "truncated_count": 0}, "processing_total": {"avg_ms": 254.184, "count": 114, "max_ms": 307.915, "p95_ms": 271.035}, "read_interval": {"avg_ms": 263.609, "count": 114, "max_ms": 316.911, "p95_ms": 280.231}, "triton_wait": {"avg_ms": 70.724, "count": 113, "max_ms": 87.59, "p95_ms": 83.912}, "visualize": {"avg_ms": 11.983, "count": 113, "max_ms": 18.106, "p95_ms": 13.663}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.857, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 10}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5114.973, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:56:53+09:00", "last_error_summary": "[h264 @ 0x562aebd74a00] error while decoding MB 74 7, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:56:53+09:00", "last_error_summary": "[h264 @ 0x562aebd74a00] error while decoding MB 74 7, bytestream -5", "last_exit_at": "2026-07-01T11:56:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.2, "last_started_at": "2026-07-01T11:55:02+09:00", "pid": 2984, "process": {"exists": false, "pid": 2984}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 305.306, "sample_count": 1095, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1235, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:56:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T11:56:53+09:00", "last_frame_read_age_sec": 0.253, "last_frame_read_at": "2026-07-01T11:56:53+09:00", "mjpeg": {"age_sec": 0.248, "exists": true, "mtime": "2026-07-01T11:56:53.336514+09:00", "mtime_epoch": 1782874613.3365135, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177134}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:56:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.027, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.825, "pushed": 3.825, "read": 3.791}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.356, "count": 114, "max_ms": 17.372, "p95_ms": 11.22}, "gstreamer_push": {"avg_ms": 0.851, "count": 114, "max_ms": 0.936, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.057, "count": 114, "max_ms": 48.227, "p95_ms": 41.393}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.293, "count": 114, "max_ms": 87.579, "p95_ms": 83.899}, "yolo11_640_fire_11class": {"avg_ms": 48.203, "count": 114, "max_ms": 61.184, "p95_ms": 58.867}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.7, "count": 114, "max_ms": 84.776, "p95_ms": 80.541}}, "truncated_count": 0}, "processing_total": {"avg_ms": 254.219, "count": 114, "max_ms": 307.915, "p95_ms": 271.035}, "read_interval": {"avg_ms": 263.694, "count": 113, "max_ms": 316.911, "p95_ms": 280.231}, "triton_wait": {"avg_ms": 70.678, "count": 114, "max_ms": 87.59, "p95_ms": 83.912}, "visualize": {"avg_ms": 11.984, "count": 114, "max_ms": 18.106, "p95_ms": 13.663}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.805, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 10}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5114.945, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:56:53+09:00", "last_error_summary": "[h264 @ 0x562aebd74a00] error while decoding MB 74 7, bytestream -5", "last_exit_at": "2026-07-01T11:56:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.203, "last_started_at": "2026-07-01T11:55:02+09:00", "pid": 2984, "process": {"exists": false, "pid": 2984}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 305.306, "sample_count": 1095, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1235, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:56:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T11:56:53+09:00", "last_frame_read_age_sec": 0.257, "last_frame_read_at": "2026-07-01T11:56:53+09:00", "mjpeg": {"age_sec": 0.251, "exists": true, "mtime": "2026-07-01T11:56:53.336514+09:00", "mtime_epoch": 1782874613.3365135, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177134}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:56:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.027, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.824, "pushed": 3.824, "read": 3.791}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.356, "count": 114, "max_ms": 17.372, "p95_ms": 11.22}, "gstreamer_push": {"avg_ms": 0.851, "count": 114, "max_ms": 0.936, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.057, "count": 114, "max_ms": 48.227, "p95_ms": 41.393}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.293, "count": 114, "max_ms": 87.579, "p95_ms": 83.899}, "yolo11_640_fire_11class": {"avg_ms": 48.203, "count": 114, "max_ms": 61.184, "p95_ms": 58.867}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.7, "count": 114, "max_ms": 84.776, "p95_ms": 80.541}}, "truncated_count": 0}, "processing_total": {"avg_ms": 254.219, "count": 114, "max_ms": 307.915, "p95_ms": 271.035}, "read_interval": {"avg_ms": 263.694, "count": 113, "max_ms": 316.911, "p95_ms": 280.231}, "triton_wait": {"avg_ms": 70.678, "count": 114, "max_ms": 87.59, "p95_ms": 83.912}, "visualize": {"avg_ms": 11.984, "count": 114, "max_ms": 18.106, "p95_ms": 13.663}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.809, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 10}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5220.129, "limit_mb": 8796093022207.996}, "detail": {"pid": 3091, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:56:53+09:00", "last_error_summary": "[h264 @ 0x562aebd74a00] error while decoding MB 74 7, bytestream -5", "last_exit_at": "2026-07-01T11:56:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:56:55+09:00", "pid": 3091, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.199, "name": "ffmpeg", "pid": 3091, "rss_mb": 139.199, "state": "S (sleeping)", "threads": 106}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 302.232, "sample_count": 1082, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1235, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:56:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.11, "last_frame_pushed_at": "2026-07-01T11:56:53+09:00", "last_frame_read_age_sec": 4.361, "last_frame_read_at": "2026-07-01T11:56:53+09:00", "mjpeg": {"age_sec": 4.095, "exists": true, "mtime": "2026-07-01T11:56:53.596518+09:00", "mtime_epoch": 1782874613.5965178, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177242}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:56:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.027, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.267, "pushed": 3.267, "read": 3.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.396, "count": 98, "max_ms": 17.372, "p95_ms": 11.356}, "gstreamer_push": {"avg_ms": 0.849, "count": 98, "max_ms": 0.936, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.288, "count": 98, "max_ms": 48.227, "p95_ms": 42.283}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.801, "count": 98, "max_ms": 87.579, "p95_ms": 83.995}, "yolo11_640_fire_11class": {"avg_ms": 48.521, "count": 98, "max_ms": 61.184, "p95_ms": 59.216}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.095, "count": 98, "max_ms": 84.776, "p95_ms": 82.111}}, "truncated_count": 0}, "processing_total": {"avg_ms": 254.812, "count": 98, "max_ms": 307.915, "p95_ms": 272.88}, "read_interval": {"avg_ms": 264.164, "count": 98, "max_ms": 316.911, "p95_ms": 281.99}, "triton_wait": {"avg_ms": 71.219, "count": 98, "max_ms": 87.59, "p95_ms": 84.007}, "visualize": {"avg_ms": 12.02, "count": 98, "max_ms": 18.106, "p95_ms": 13.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5203.18, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[NULL @ 0x564de15bb300] missing picture in access unit with size 39 [h264 @ 0x564de16f2840] no frame! [vist#0:0/h264 @ 0x564de15c12c0] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:58:47+09:00", "last_error_summary": "[NULL @ 0x564de15bb300] missing picture in access unit with size 39 [h264 @ 0x564de16f2840] no frame! [vist#0:0/h264 @ 0x564de15c12c0] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T11:56:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.276, "last_started_at": "2026-07-01T11:56:55+09:00", "pid": 3091, "process": {"exists": false, "pid": 3091}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.262, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:58:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-01T11:58:47+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-01T11:58:47+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T11:58:47.926453+09:00", "mtime_epoch": 1782874727.9264526, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177518}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:58:47+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.285, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.883, "pushed": 3.883, "read": 3.883}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.542, "count": 115, "max_ms": 15.869, "p95_ms": 14.045}, "gstreamer_push": {"avg_ms": 0.846, "count": 116, "max_ms": 1.076, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.586, "count": 115, "max_ms": 52.895, "p95_ms": 43.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.745, "count": 115, "max_ms": 83.254, "p95_ms": 80.573}, "yolo11_640_fire_11class": {"avg_ms": 44.743, "count": 115, "max_ms": 60.998, "p95_ms": 57.54}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.219, "count": 115, "max_ms": 79.811, "p95_ms": 76.002}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.791, "count": 116, "max_ms": 271.178, "p95_ms": 264.902}, "read_interval": {"avg_ms": 259.247, "count": 116, "max_ms": 280.104, "p95_ms": 273.956}, "triton_wait": {"avg_ms": 67.049, "count": 115, "max_ms": 83.269, "p95_ms": 80.588}, "visualize": {"avg_ms": 11.75, "count": 115, "max_ms": 15.865, "p95_ms": 12.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5202.418, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T11:58:47+09:00", "last_error_summary": "[NULL @ 0x564de15bb300] missing picture in access unit with size 39 [h264 @ 0x564de16f2840] no frame! [vist#0:0/h264 @ 0x564de15c12c0] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:58:47+09:00", "last_error_summary": "[NULL @ 0x564de15bb300] missing picture in access unit with size 39 [h264 @ 0x564de16f2840] no frame! [vist#0:0/h264 @ 0x564de15c12c0] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T11:58:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.488, "last_started_at": "2026-07-01T11:56:55+09:00", "pid": 3091, "process": {"exists": false, "pid": 3091}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.262, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:58:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T11:58:48+09:00", "last_frame_read_age_sec": 0.26, "last_frame_read_at": "2026-07-01T11:58:47+09:00", "mjpeg": {"age_sec": 0.253, "exists": true, "mtime": "2026-07-01T11:58:47.926453+09:00", "mtime_epoch": 1782874727.9264526, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177518}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:58:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.889, "pushed": 3.889, "read": 3.855}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.542, "count": 116, "max_ms": 15.869, "p95_ms": 14.045}, "gstreamer_push": {"avg_ms": 0.846, "count": 116, "max_ms": 1.076, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.569, "count": 116, "max_ms": 52.895, "p95_ms": 43.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.796, "count": 116, "max_ms": 83.254, "p95_ms": 80.573}, "yolo11_640_fire_11class": {"avg_ms": 44.745, "count": 116, "max_ms": 60.998, "p95_ms": 57.54}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.217, "count": 116, "max_ms": 79.811, "p95_ms": 76.002}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.847, "count": 116, "max_ms": 271.178, "p95_ms": 264.902}, "read_interval": {"avg_ms": 259.266, "count": 115, "max_ms": 280.104, "p95_ms": 273.956}, "triton_wait": {"avg_ms": 67.098, "count": 116, "max_ms": 83.269, "p95_ms": 80.588}, "visualize": {"avg_ms": 11.751, "count": 116, "max_ms": 15.865, "p95_ms": 12.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.828, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5202.41, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:58:47+09:00", "last_error_summary": "[NULL @ 0x564de15bb300] missing picture in access unit with size 39 [h264 @ 0x564de16f2840] no frame! [vist#0:0/h264 @ 0x564de15c12c0] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T11:58:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.491, "last_started_at": "2026-07-01T11:56:55+09:00", "pid": 3091, "process": {"exists": false, "pid": 3091}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.262, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:58:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T11:58:48+09:00", "last_frame_read_age_sec": 0.263, "last_frame_read_at": "2026-07-01T11:58:47+09:00", "mjpeg": {"age_sec": 0.256, "exists": true, "mtime": "2026-07-01T11:58:47.926453+09:00", "mtime_epoch": 1782874727.9264526, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177518}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:58:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.889, "pushed": 3.889, "read": 3.855}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.542, "count": 116, "max_ms": 15.869, "p95_ms": 14.045}, "gstreamer_push": {"avg_ms": 0.846, "count": 116, "max_ms": 1.076, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.569, "count": 116, "max_ms": 52.895, "p95_ms": 43.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.796, "count": 116, "max_ms": 83.254, "p95_ms": 80.573}, "yolo11_640_fire_11class": {"avg_ms": 44.745, "count": 116, "max_ms": 60.998, "p95_ms": 57.54}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.217, "count": 116, "max_ms": 79.811, "p95_ms": 76.002}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.847, "count": 116, "max_ms": 271.178, "p95_ms": 264.902}, "read_interval": {"avg_ms": 259.266, "count": 115, "max_ms": 280.104, "p95_ms": 273.956}, "triton_wait": {"avg_ms": 67.098, "count": 116, "max_ms": 83.269, "p95_ms": 80.588}, "visualize": {"avg_ms": 11.751, "count": 116, "max_ms": 15.865, "p95_ms": 12.505}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.831, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 13}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5311.734, "limit_mb": 8796093022207.996}, "detail": {"pid": 3198, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T11:58:47+09:00", "last_error_summary": "[NULL @ 0x564de15bb300] missing picture in access unit with size 39 [h264 @ 0x564de16f2840] no frame! [vist#0:0/h264 @ 0x564de15c12c0] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T11:58:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T11:58:50+09:00", "pid": 3198, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.184, "name": "ffmpeg", "pid": 3198, "rss_mb": 136.184, "state": "R (running)", "threads": 106}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.267, "sample_count": 1106, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T11:58:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.187, "last_frame_pushed_at": "2026-07-01T11:58:48+09:00", "last_frame_read_age_sec": 4.442, "last_frame_read_at": "2026-07-01T11:58:47+09:00", "mjpeg": {"age_sec": 4.171, "exists": true, "mtime": "2026-07-01T11:58:48.190457+09:00", "mtime_epoch": 1782874728.190457, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177626}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T11:58:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.285, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.349, "pushed": 3.349, "read": 3.316}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.503, "count": 99, "max_ms": 15.869, "p95_ms": 14.168}, "gstreamer_push": {"avg_ms": 0.845, "count": 100, "max_ms": 1.076, "p95_ms": 0.878}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.35, "count": 99, "max_ms": 52.895, "p95_ms": 43.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.515, "count": 99, "max_ms": 83.254, "p95_ms": 80.753}, "yolo11_640_fire_11class": {"avg_ms": 44.238, "count": 99, "max_ms": 60.998, "p95_ms": 57.54}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.911, "count": 99, "max_ms": 77.238, "p95_ms": 76.002}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.876, "count": 100, "max_ms": 271.178, "p95_ms": 264.902}, "read_interval": {"avg_ms": 259.301, "count": 99, "max_ms": 280.104, "p95_ms": 273.964}, "triton_wait": {"avg_ms": 66.816, "count": 99, "max_ms": 83.269, "p95_ms": 80.765}, "visualize": {"avg_ms": 11.759, "count": 99, "max_ms": 15.865, "p95_ms": 12.816}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.859, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5379.141, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5555ec08c000] error while decoding MB 27 17, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:00:43+09:00", "last_error_summary": "[h264 @ 0x5555ec08c000] error while decoding MB 27 17, bytestream -5", "last_exit_at": "2026-07-01T11:58:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.568, "last_started_at": "2026-07-01T11:58:50+09:00", "pid": 3198, "process": {"exists": false, "pid": 3198}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.529, "sample_count": 1121, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:00:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T12:00:43+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T12:00:43+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T12:00:43.888413+09:00", "mtime_epoch": 1782874843.8884127, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178646}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:00:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.543, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.9, "pushed": 3.9, "read": 3.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.436, "count": 116, "max_ms": 14.479, "p95_ms": 11.988}, "gstreamer_push": {"avg_ms": 0.843, "count": 117, "max_ms": 0.987, "p95_ms": 0.893}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.351, "count": 116, "max_ms": 47.793, "p95_ms": 42.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.305, "count": 116, "max_ms": 84.434, "p95_ms": 81.272}, "yolo11_640_fire_11class": {"avg_ms": 45.967, "count": 116, "max_ms": 61.388, "p95_ms": 57.459}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.757, "count": 116, "max_ms": 80.447, "p95_ms": 77.054}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.967, "count": 117, "max_ms": 268.191, "p95_ms": 265.389}, "read_interval": {"avg_ms": 258.173, "count": 117, "max_ms": 276.26, "p95_ms": 274.322}, "triton_wait": {"avg_ms": 66.033, "count": 116, "max_ms": 84.446, "p95_ms": 81.284}, "visualize": {"avg_ms": 11.975, "count": 116, "max_ms": 16.942, "p95_ms": 13.955}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5378.047, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:00:43+09:00", "last_error_summary": "[h264 @ 0x5555ec08c000] error while decoding MB 27 17, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:00:43+09:00", "last_error_summary": "[h264 @ 0x5555ec08c000] error while decoding MB 27 17, bytestream -5", "last_exit_at": "2026-07-01T12:00:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.764, "last_started_at": "2026-07-01T11:58:50+09:00", "pid": 3198, "process": {"exists": false, "pid": 3198}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.529, "sample_count": 1121, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:00:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:00:44+09:00", "last_frame_read_age_sec": 0.245, "last_frame_read_at": "2026-07-01T12:00:43+09:00", "mjpeg": {"age_sec": 0.236, "exists": true, "mtime": "2026-07-01T12:00:43.888413+09:00", "mtime_epoch": 1782874843.8884127, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178646}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:00:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.543, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.908, "pushed": 3.908, "read": 3.875}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.435, "count": 116, "max_ms": 14.479, "p95_ms": 11.988}, "gstreamer_push": {"avg_ms": 0.843, "count": 117, "max_ms": 0.987, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.322, "count": 116, "max_ms": 47.793, "p95_ms": 42.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.259, "count": 116, "max_ms": 84.434, "p95_ms": 81.272}, "yolo11_640_fire_11class": {"avg_ms": 45.901, "count": 116, "max_ms": 61.388, "p95_ms": 57.459}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.693, "count": 116, "max_ms": 80.447, "p95_ms": 77.054}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.816, "count": 117, "max_ms": 268.191, "p95_ms": 265.389}, "read_interval": {"avg_ms": 258.094, "count": 116, "max_ms": 276.26, "p95_ms": 274.322}, "triton_wait": {"avg_ms": 65.979, "count": 116, "max_ms": 84.446, "p95_ms": 81.284}, "visualize": {"avg_ms": 11.97, "count": 116, "max_ms": 16.942, "p95_ms": 13.955}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5378.164, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:00:43+09:00", "last_error_summary": "[h264 @ 0x5555ec08c000] error while decoding MB 27 17, bytestream -5", "last_exit_at": "2026-07-01T12:00:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.767, "last_started_at": "2026-07-01T11:58:50+09:00", "pid": 3198, "process": {"exists": false, "pid": 3198}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.529, "sample_count": 1121, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:00:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:00:44+09:00", "last_frame_read_age_sec": 0.248, "last_frame_read_at": "2026-07-01T12:00:43+09:00", "mjpeg": {"age_sec": 0.239, "exists": true, "mtime": "2026-07-01T12:00:43.888413+09:00", "mtime_epoch": 1782874843.8884127, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178646}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:00:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.543, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.908, "pushed": 3.908, "read": 3.874}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.435, "count": 116, "max_ms": 14.479, "p95_ms": 11.988}, "gstreamer_push": {"avg_ms": 0.843, "count": 117, "max_ms": 0.987, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.322, "count": 116, "max_ms": 47.793, "p95_ms": 42.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.259, "count": 116, "max_ms": 84.434, "p95_ms": 81.272}, "yolo11_640_fire_11class": {"avg_ms": 45.901, "count": 116, "max_ms": 61.388, "p95_ms": 57.459}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.693, "count": 116, "max_ms": 80.447, "p95_ms": 77.054}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.816, "count": 117, "max_ms": 268.191, "p95_ms": 265.389}, "read_interval": {"avg_ms": 258.094, "count": 116, "max_ms": 276.26, "p95_ms": 274.322}, "triton_wait": {"avg_ms": 65.979, "count": 116, "max_ms": 84.446, "p95_ms": 81.284}, "visualize": {"avg_ms": 11.97, "count": 116, "max_ms": 16.942, "p95_ms": 13.955}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 17}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5489.191, "limit_mb": 8796093022207.996}, "detail": {"pid": 3366, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:00:43+09:00", "last_error_summary": "[h264 @ 0x5555ec08c000] error while decoding MB 27 17, bytestream -5", "last_exit_at": "2026-07-01T12:00:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:00:46+09:00", "pid": 3366, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.426, "name": "ffmpeg", "pid": 3366, "rss_mb": 141.426, "state": "S (sleeping)", "threads": 106}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.586, "sample_count": 1106, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:00:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.044, "last_frame_pushed_at": "2026-07-01T12:00:44+09:00", "last_frame_read_age_sec": 4.286, "last_frame_read_at": "2026-07-01T12:00:43+09:00", "mjpeg": {"age_sec": 4.029, "exists": true, "mtime": "2026-07-01T12:00:44.136417+09:00", "mtime_epoch": 1782874844.136417, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178596}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:00:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.543, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.379, "pushed": 3.379, "read": 3.346}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.427, "count": 100, "max_ms": 14.282, "p95_ms": 11.604}, "gstreamer_push": {"avg_ms": 0.844, "count": 101, "max_ms": 0.987, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.287, "count": 100, "max_ms": 47.793, "p95_ms": 41.062}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.8, "count": 100, "max_ms": 84.434, "p95_ms": 81.272}, "yolo11_640_fire_11class": {"avg_ms": 46.018, "count": 100, "max_ms": 61.388, "p95_ms": 57.459}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.789, "count": 100, "max_ms": 80.447, "p95_ms": 77.054}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.155, "count": 101, "max_ms": 268.191, "p95_ms": 265.389}, "read_interval": {"avg_ms": 258.404, "count": 100, "max_ms": 276.26, "p95_ms": 274.322}, "triton_wait": {"avg_ms": 66.4, "count": 100, "max_ms": 84.446, "p95_ms": 81.284}, "visualize": {"avg_ms": 12.008, "count": 100, "max_ms": 16.942, "p95_ms": 13.955}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5584.562, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5620b023cdc0] error while decoding MB 78 19, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:02:40+09:00", "last_error_summary": "[h264 @ 0x5620b023cdc0] error while decoding MB 78 19, bytestream -14", "last_exit_at": "2026-07-01T12:00:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.294, "last_started_at": "2026-07-01T12:00:46+09:00", "pid": 3366, "process": {"exists": false, "pid": 3366}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.058, "sample_count": 1112, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2510, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:02:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T12:02:40+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T12:02:40+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T12:02:40.414380+09:00", "mtime_epoch": 1782874960.4143803, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172454}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:02:40+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.801, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.884, "pushed": 3.884, "read": 3.884}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.188, "count": 115, "max_ms": 14.818, "p95_ms": 11.303}, "gstreamer_push": {"avg_ms": 0.847, "count": 116, "max_ms": 1.125, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.52, "count": 115, "max_ms": 51.203, "p95_ms": 44.655}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.033, "count": 115, "max_ms": 85.274, "p95_ms": 81.619}, "yolo11_640_fire_11class": {"avg_ms": 48.272, "count": 115, "max_ms": 61.671, "p95_ms": 56.094}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.722, "count": 115, "max_ms": 85.82, "p95_ms": 80.644}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.703, "count": 116, "max_ms": 301.687, "p95_ms": 265.395}, "read_interval": {"avg_ms": 259.18, "count": 116, "max_ms": 312.453, "p95_ms": 275.514}, "triton_wait": {"avg_ms": 68.151, "count": 115, "max_ms": 85.832, "p95_ms": 82.092}, "visualize": {"avg_ms": 11.738, "count": 115, "max_ms": 18.093, "p95_ms": 13.235}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.865, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5583.984, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:02:40+09:00", "last_error_summary": "[h264 @ 0x5620b023cdc0] error while decoding MB 78 19, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:02:40+09:00", "last_error_summary": "[h264 @ 0x5620b023cdc0] error while decoding MB 78 19, bytestream -14", "last_exit_at": "2026-07-01T12:02:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.488, "last_started_at": "2026-07-01T12:00:46+09:00", "pid": 3366, "process": {"exists": false, "pid": 3366}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.058, "sample_count": 1112, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2511, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:02:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:02:40+09:00", "last_frame_read_age_sec": 0.244, "last_frame_read_at": "2026-07-01T12:02:40+09:00", "mjpeg": {"age_sec": 0.238, "exists": true, "mtime": "2026-07-01T12:02:40.414380+09:00", "mtime_epoch": 1782874960.4143803, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172454}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:02:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.801, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.892, "pushed": 3.892, "read": 3.859}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.19, "count": 116, "max_ms": 14.818, "p95_ms": 11.303}, "gstreamer_push": {"avg_ms": 0.847, "count": 116, "max_ms": 1.125, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.485, "count": 116, "max_ms": 51.203, "p95_ms": 44.655}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.984, "count": 116, "max_ms": 85.274, "p95_ms": 81.619}, "yolo11_640_fire_11class": {"avg_ms": 48.25, "count": 116, "max_ms": 61.671, "p95_ms": 56.094}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.668, "count": 116, "max_ms": 85.82, "p95_ms": 80.644}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.619, "count": 116, "max_ms": 301.687, "p95_ms": 265.395}, "read_interval": {"avg_ms": 259.178, "count": 115, "max_ms": 312.453, "p95_ms": 275.514}, "triton_wait": {"avg_ms": 68.092, "count": 116, "max_ms": 85.832, "p95_ms": 82.092}, "visualize": {"avg_ms": 11.737, "count": 116, "max_ms": 18.093, "p95_ms": 13.235}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.802, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5583.977, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:02:40+09:00", "last_error_summary": "[h264 @ 0x5620b023cdc0] error while decoding MB 78 19, bytestream -14", "last_exit_at": "2026-07-01T12:02:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.491, "last_started_at": "2026-07-01T12:00:46+09:00", "pid": 3366, "process": {"exists": false, "pid": 3366}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.058, "sample_count": 1112, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2511, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:02:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:02:40+09:00", "last_frame_read_age_sec": 0.247, "last_frame_read_at": "2026-07-01T12:02:40+09:00", "mjpeg": {"age_sec": 0.241, "exists": true, "mtime": "2026-07-01T12:02:40.414380+09:00", "mtime_epoch": 1782874960.4143803, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172454}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:02:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.801, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.892, "pushed": 3.892, "read": 3.858}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.19, "count": 116, "max_ms": 14.818, "p95_ms": 11.303}, "gstreamer_push": {"avg_ms": 0.847, "count": 116, "max_ms": 1.125, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.485, "count": 116, "max_ms": 51.203, "p95_ms": 44.655}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.984, "count": 116, "max_ms": 85.274, "p95_ms": 81.619}, "yolo11_640_fire_11class": {"avg_ms": 48.25, "count": 116, "max_ms": 61.671, "p95_ms": 56.094}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.668, "count": 116, "max_ms": 85.82, "p95_ms": 80.644}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.619, "count": 116, "max_ms": 301.687, "p95_ms": 265.395}, "read_interval": {"avg_ms": 259.178, "count": 115, "max_ms": 312.453, "p95_ms": 275.514}, "triton_wait": {"avg_ms": 68.092, "count": 116, "max_ms": 85.832, "p95_ms": 82.092}, "visualize": {"avg_ms": 11.737, "count": 116, "max_ms": 18.093, "p95_ms": 13.235}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.805, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5690.742, "limit_mb": 8796093022207.996}, "detail": {"pid": 3593, "restart_count": 6, "start_count": 7}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:02:40+09:00", "last_error_summary": "[h264 @ 0x5620b023cdc0] error while decoding MB 78 19, bytestream -14", "last_exit_at": "2026-07-01T12:02:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:02:42+09:00", "pid": 3593, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.484, "name": "ffmpeg", "pid": 3593, "rss_mb": 139.484, "state": "S (sleeping)", "threads": 106}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.058, "sample_count": 1096, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2511, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:02:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.192, "last_frame_pushed_at": "2026-07-01T12:02:40+09:00", "last_frame_read_age_sec": 4.434, "last_frame_read_at": "2026-07-01T12:02:40+09:00", "mjpeg": {"age_sec": 4.176, "exists": true, "mtime": "2026-07-01T12:02:40.666385+09:00", "mtime_epoch": 1782874960.6663847, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170894}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:02:44+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1907.801, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.356, "pushed": 3.356, "read": 3.322}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.166, "count": 100, "max_ms": 13.449, "p95_ms": 11.224}, "gstreamer_push": {"avg_ms": 0.845, "count": 100, "max_ms": 1.087, "p95_ms": 0.892}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.342, "count": 100, "max_ms": 46.984, "p95_ms": 42.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.964, "count": 100, "max_ms": 85.274, "p95_ms": 81.619}, "yolo11_640_fire_11class": {"avg_ms": 48.177, "count": 100, "max_ms": 61.671, "p95_ms": 56.094}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.655, "count": 100, "max_ms": 85.82, "p95_ms": 80.644}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.135, "count": 100, "max_ms": 267.012, "p95_ms": 265.242}, "read_interval": {"avg_ms": 258.677, "count": 99, "max_ms": 277.055, "p95_ms": 275.514}, "triton_wait": {"avg_ms": 68.134, "count": 100, "max_ms": 85.832, "p95_ms": 82.092}, "visualize": {"avg_ms": 11.748, "count": 100, "max_ms": 18.093, "p95_ms": 12.9}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.799, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 24}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5584.227, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:02:40+09:00", "last_error_summary": "[h264 @ 0x5620b023cdc0] error while decoding MB 78 19, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:02:40+09:00", "last_error_summary": "[h264 @ 0x5620b023cdc0] error while decoding MB 78 19, bytestream -14", "last_exit_at": "2026-07-01T12:04:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.9, "last_started_at": "2026-07-01T12:02:42+09:00", "pid": 3593, "process": {"exists": false, "pid": 3593}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 279.384, "sample_count": 1096, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2930, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:04:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:04:37+09:00", "last_frame_read_age_sec": 0.289, "last_frame_read_at": "2026-07-01T12:04:37+09:00", "mjpeg": {"age_sec": 0.281, "exists": true, "mtime": "2026-07-01T12:04:37.460335+09:00", "mtime_epoch": 1782875077.4603353, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171996}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:04:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1908.059, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.741, "pushed": 3.741, "read": 3.707}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.95, "count": 111, "max_ms": 15.372, "p95_ms": 11.891}, "gstreamer_push": {"avg_ms": 0.851, "count": 112, "max_ms": 1.276, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.143, "count": 111, "max_ms": 46.742, "p95_ms": 39.868}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.305, "count": 111, "max_ms": 83.635, "p95_ms": 82.449}, "yolo11_640_fire_11class": {"avg_ms": 47.927, "count": 111, "max_ms": 64.961, "p95_ms": 59.251}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.057, "count": 111, "max_ms": 83.653, "p95_ms": 81.247}}, "truncated_count": 0}, "processing_total": {"avg_ms": 260.052, "count": 112, "max_ms": 337.422, "p95_ms": 284.174}, "read_interval": {"avg_ms": 269.309, "count": 111, "max_ms": 347.98, "p95_ms": 294.211}, "triton_wait": {"avg_ms": 69.616, "count": 111, "max_ms": 83.664, "p95_ms": 83.206}, "visualize": {"avg_ms": 12.147, "count": 111, "max_ms": 17.271, "p95_ms": 13.399}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 24}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5584.219, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:02:40+09:00", "last_error_summary": "[h264 @ 0x5620b023cdc0] error while decoding MB 78 19, bytestream -14", "last_exit_at": "2026-07-01T12:04:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.903, "last_started_at": "2026-07-01T12:02:42+09:00", "pid": 3593, "process": {"exists": false, "pid": 3593}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 279.384, "sample_count": 1096, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2930, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:04:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:04:37+09:00", "last_frame_read_age_sec": 0.293, "last_frame_read_at": "2026-07-01T12:04:37+09:00", "mjpeg": {"age_sec": 0.284, "exists": true, "mtime": "2026-07-01T12:04:37.460335+09:00", "mtime_epoch": 1782875077.4603353, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171996}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:04:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1908.059, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.741, "pushed": 3.741, "read": 3.707}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.95, "count": 111, "max_ms": 15.372, "p95_ms": 11.891}, "gstreamer_push": {"avg_ms": 0.851, "count": 112, "max_ms": 1.276, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.143, "count": 111, "max_ms": 46.742, "p95_ms": 39.868}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.305, "count": 111, "max_ms": 83.635, "p95_ms": 82.449}, "yolo11_640_fire_11class": {"avg_ms": 47.927, "count": 111, "max_ms": 64.961, "p95_ms": 59.251}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.057, "count": 111, "max_ms": 83.653, "p95_ms": 81.247}}, "truncated_count": 0}, "processing_total": {"avg_ms": 260.052, "count": 112, "max_ms": 337.422, "p95_ms": 284.174}, "read_interval": {"avg_ms": 269.309, "count": 111, "max_ms": 347.98, "p95_ms": 294.211}, "triton_wait": {"avg_ms": 69.616, "count": 111, "max_ms": 83.664, "p95_ms": 83.206}, "visualize": {"avg_ms": 12.147, "count": 111, "max_ms": 17.271, "p95_ms": 13.399}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 24}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5691.734, "limit_mb": 8796093022207.996}, "detail": {"pid": 3700, "restart_count": 7, "start_count": 8}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:02:40+09:00", "last_error_summary": "[h264 @ 0x5620b023cdc0] error while decoding MB 78 19, bytestream -14", "last_exit_at": "2026-07-01T12:04:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:04:39+09:00", "pid": 3700, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.832, "name": "ffmpeg", "pid": 3700, "rss_mb": 138.832, "state": "S (sleeping)", "threads": 106}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 279.512, "sample_count": 1079, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2930, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:04:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.099, "last_frame_pushed_at": "2026-07-01T12:04:37+09:00", "last_frame_read_age_sec": 4.386, "last_frame_read_at": "2026-07-01T12:04:37+09:00", "mjpeg": {"age_sec": 4.085, "exists": true, "mtime": "2026-07-01T12:04:37.752340+09:00", "mtime_epoch": 1782875077.7523396, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:04:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1942.438, "name": "celery", "pid": 237, "rss_mb": 1908.059, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.213, "pushed": 3.213, "read": 3.179}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.009, "count": 95, "max_ms": 15.372, "p95_ms": 12.005}, "gstreamer_push": {"avg_ms": 0.852, "count": 96, "max_ms": 1.276, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.952, "count": 95, "max_ms": 46.742, "p95_ms": 41.762}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.371, "count": 95, "max_ms": 83.635, "p95_ms": 82.938}, "yolo11_640_fire_11class": {"avg_ms": 47.868, "count": 95, "max_ms": 64.961, "p95_ms": 59.87}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.707, "count": 95, "max_ms": 83.653, "p95_ms": 81.382}}, "truncated_count": 0}, "processing_total": {"avg_ms": 261.626, "count": 96, "max_ms": 337.422, "p95_ms": 285.197}, "read_interval": {"avg_ms": 270.87, "count": 95, "max_ms": 347.98, "p95_ms": 294.658}, "triton_wait": {"avg_ms": 69.671, "count": 95, "max_ms": 83.664, "p95_ms": 83.474}, "visualize": {"avg_ms": 12.247, "count": 95, "max_ms": 17.271, "p95_ms": 13.488}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 5, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 26}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5854.957, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56283db1c240] error while decoding MB 34 33, bytestream -21", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:06:56+09:00", "last_error_summary": "[h264 @ 0x56283db1c240] error while decoding MB 34 33, bytestream -21", "last_exit_at": "2026-07-01T12:04:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.551, "last_started_at": "2026-07-01T12:04:39+09:00", "pid": 3700, "process": {"exists": false, "pid": 3700}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 511.098, "sample_count": 876, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3216, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:06:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-01T12:06:56+09:00", "last_frame_read_age_sec": 0.047, "last_frame_read_at": "2026-07-01T12:06:56+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T12:06:56.350444+09:00", "mtime_epoch": 1782875216.3504443, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175904}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:06:56+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2346.293, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 59, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.985, "pushed": 1.985, "read": 1.985}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.435, "count": 58, "max_ms": 24.69, "p95_ms": 23.349}, "gstreamer_push": {"avg_ms": 1.12, "count": 59, "max_ms": 2.909, "p95_ms": 1.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.307, "count": 58, "max_ms": 49.166, "p95_ms": 46.028}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.924, "count": 58, "max_ms": 94.161, "p95_ms": 89.901}, "yolo11_640_fire_11class": {"avg_ms": 47.127, "count": 58, "max_ms": 58.221, "p95_ms": 56.768}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.137, "count": 58, "max_ms": 88.601, "p95_ms": 87.369}}, "truncated_count": 0}, "processing_total": {"avg_ms": 500.521, "count": 59, "max_ms": 1032.547, "p95_ms": 679.063}, "read_interval": {"avg_ms": 510.739, "count": 59, "max_ms": 1044.718, "p95_ms": 689.319}, "triton_wait": {"avg_ms": 78.296, "count": 58, "max_ms": 94.174, "p95_ms": 89.913}, "visualize": {"avg_ms": 23.377, "count": 58, "max_ms": 33.316, "p95_ms": 31.034}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.718, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 5, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 26}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5853.762, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:06:56+09:00", "last_error_summary": "[h264 @ 0x56283db1c240] error while decoding MB 34 33, bytestream -21", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:06:56+09:00", "last_error_summary": "[h264 @ 0x56283db1c240] error while decoding MB 34 33, bytestream -21", "last_exit_at": "2026-07-01T12:06:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.905, "last_started_at": "2026-07-01T12:04:39+09:00", "pid": 3700, "process": {"exists": false, "pid": 3700}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 511.098, "sample_count": 876, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:06:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:06:56+09:00", "last_frame_read_age_sec": 0.401, "last_frame_read_at": "2026-07-01T12:06:56+09:00", "mjpeg": {"age_sec": 0.391, "exists": true, "mtime": "2026-07-01T12:06:56.350444+09:00", "mtime_epoch": 1782875216.3504443, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175904}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:06:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2346.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.994, "pushed": 1.994, "read": 1.961}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.459, "count": 59, "max_ms": 24.69, "p95_ms": 23.349}, "gstreamer_push": {"avg_ms": 1.121, "count": 59, "max_ms": 2.909, "p95_ms": 1.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.245, "count": 59, "max_ms": 49.166, "p95_ms": 46.028}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.785, "count": 59, "max_ms": 94.161, "p95_ms": 89.901}, "yolo11_640_fire_11class": {"avg_ms": 47.193, "count": 59, "max_ms": 58.221, "p95_ms": 56.768}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.973, "count": 59, "max_ms": 88.601, "p95_ms": 87.369}}, "truncated_count": 0}, "processing_total": {"avg_ms": 499.394, "count": 59, "max_ms": 1032.547, "p95_ms": 679.063}, "read_interval": {"avg_ms": 511.392, "count": 58, "max_ms": 1044.718, "p95_ms": 689.319}, "triton_wait": {"avg_ms": 78.15, "count": 59, "max_ms": 94.174, "p95_ms": 89.913}, "visualize": {"avg_ms": 23.313, "count": 59, "max_ms": 33.316, "p95_ms": 31.034}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.584, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 5, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 26}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5853.754, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:06:56+09:00", "last_error_summary": "[h264 @ 0x56283db1c240] error while decoding MB 34 33, bytestream -21", "last_exit_at": "2026-07-01T12:06:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.908, "last_started_at": "2026-07-01T12:04:39+09:00", "pid": 3700, "process": {"exists": false, "pid": 3700}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 511.098, "sample_count": 876, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:06:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:06:56+09:00", "last_frame_read_age_sec": 0.404, "last_frame_read_at": "2026-07-01T12:06:56+09:00", "mjpeg": {"age_sec": 0.394, "exists": true, "mtime": "2026-07-01T12:06:56.350444+09:00", "mtime_epoch": 1782875216.3504443, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175904}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:06:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2346.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.994, "pushed": 1.994, "read": 1.96}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.459, "count": 59, "max_ms": 24.69, "p95_ms": 23.349}, "gstreamer_push": {"avg_ms": 1.121, "count": 59, "max_ms": 2.909, "p95_ms": 1.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 36.245, "count": 59, "max_ms": 49.166, "p95_ms": 46.028}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.785, "count": 59, "max_ms": 94.161, "p95_ms": 89.901}, "yolo11_640_fire_11class": {"avg_ms": 47.193, "count": 59, "max_ms": 58.221, "p95_ms": 56.768}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.973, "count": 59, "max_ms": 88.601, "p95_ms": 87.369}}, "truncated_count": 0}, "processing_total": {"avg_ms": 499.394, "count": 59, "max_ms": 1032.547, "p95_ms": 679.063}, "read_interval": {"avg_ms": 511.392, "count": 58, "max_ms": 1044.718, "p95_ms": 689.319}, "triton_wait": {"avg_ms": 78.15, "count": 59, "max_ms": 94.174, "p95_ms": 89.913}, "visualize": {"avg_ms": 23.313, "count": 59, "max_ms": 33.316, "p95_ms": 31.034}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.587, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 5, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 26}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5961.668, "limit_mb": 8796093022207.996}, "detail": {"pid": 3928, "restart_count": 8, "start_count": 9}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:06:56+09:00", "last_error_summary": "[h264 @ 0x56283db1c240] error while decoding MB 34 33, bytestream -21", "last_exit_at": "2026-07-01T12:06:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:06:59+09:00", "pid": 3928, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.277, "name": "ffmpeg", "pid": 3928, "rss_mb": 139.277, "state": "S (sleeping)", "threads": 106}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 514.434, "sample_count": 857, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:06:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.976, "last_frame_pushed_at": "2026-07-01T12:06:56+09:00", "last_frame_read_age_sec": 5.375, "last_frame_read_at": "2026-07-01T12:06:56+09:00", "mjpeg": {"age_sec": 4.96, "exists": true, "mtime": "2026-07-01T12:06:56.754451+09:00", "mtime_epoch": 1782875216.7544506, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176339}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:07:01+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2346.293, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.71, "pushed": 1.71, "read": 1.676}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.318, "count": 50, "max_ms": 23.444, "p95_ms": 22.7}, "gstreamer_push": {"avg_ms": 1.086, "count": 51, "max_ms": 2.872, "p95_ms": 1.832}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 35.405, "count": 50, "max_ms": 46.028, "p95_ms": 45.657}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.167, "count": 50, "max_ms": 90.767, "p95_ms": 89.415}, "yolo11_640_fire_11class": {"avg_ms": 46.375, "count": 50, "max_ms": 56.464, "p95_ms": 55.767}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.565, "count": 50, "max_ms": 88.601, "p95_ms": 87.369}}, "truncated_count": 0}, "processing_total": {"avg_ms": 490.53, "count": 51, "max_ms": 679.276, "p95_ms": 639.967}, "read_interval": {"avg_ms": 502.675, "count": 50, "max_ms": 691.672, "p95_ms": 651.231}, "triton_wait": {"avg_ms": 77.596, "count": 50, "max_ms": 90.775, "p95_ms": 89.427}, "visualize": {"avg_ms": 23.082, "count": 50, "max_ms": 31.304, "p95_ms": 29.951}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.829, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 30}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5855.199, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555d37690b40] error while decoding MB 6 32, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:08:50+09:00", "last_error_summary": "[h264 @ 0x555d37690b40] error while decoding MB 6 32, bytestream -7", "last_exit_at": "2026-07-01T12:06:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.958, "last_started_at": "2026-07-01T12:06:59+09:00", "pid": 3928, "process": {"exists": false, "pid": 3928}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 511.098, "sample_count": 880, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3633, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:08:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T12:08:50+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T12:08:50+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T12:08:50.632216+09:00", "mtime_epoch": 1782875330.6322157, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174401}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:08:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2346.293, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.84, "pushed": 3.84, "read": 3.84}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.993, "count": 114, "max_ms": 15.361, "p95_ms": 14.187}, "gstreamer_push": {"avg_ms": 0.863, "count": 115, "max_ms": 1.235, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.268, "count": 114, "max_ms": 48.438, "p95_ms": 43.142}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.536, "count": 114, "max_ms": 88.829, "p95_ms": 85.032}, "yolo11_640_fire_11class": {"avg_ms": 46.925, "count": 114, "max_ms": 64.958, "p95_ms": 57.852}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.225, "count": 114, "max_ms": 89.616, "p95_ms": 84.979}}, "truncated_count": 0}, "processing_total": {"avg_ms": 252.518, "count": 115, "max_ms": 306.853, "p95_ms": 275.842}, "read_interval": {"avg_ms": 262.045, "count": 115, "max_ms": 316.648, "p95_ms": 286.179}, "triton_wait": {"avg_ms": 68.223, "count": 114, "max_ms": 89.627, "p95_ms": 86.583}, "visualize": {"avg_ms": 11.794, "count": 114, "max_ms": 15.349, "p95_ms": 12.904}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 30}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5854.168, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:08:50+09:00", "last_error_summary": "[h264 @ 0x555d37690b40] error while decoding MB 6 32, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:08:50+09:00", "last_error_summary": "[h264 @ 0x555d37690b40] error while decoding MB 6 32, bytestream -7", "last_exit_at": "2026-07-01T12:08:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.171, "last_started_at": "2026-07-01T12:06:59+09:00", "pid": 3928, "process": {"exists": false, "pid": 3928}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 511.098, "sample_count": 880, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3634, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:08:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:08:50+09:00", "last_frame_read_age_sec": 0.262, "last_frame_read_at": "2026-07-01T12:08:50+09:00", "mjpeg": {"age_sec": 0.252, "exists": true, "mtime": "2026-07-01T12:08:50.632216+09:00", "mtime_epoch": 1782875330.6322157, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174401}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:08:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2346.297, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.845, "pushed": 3.845, "read": 3.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.964, "count": 114, "max_ms": 15.361, "p95_ms": 13.702}, "gstreamer_push": {"avg_ms": 0.862, "count": 115, "max_ms": 1.235, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.391, "count": 114, "max_ms": 48.438, "p95_ms": 43.142}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.701, "count": 114, "max_ms": 88.829, "p95_ms": 85.032}, "yolo11_640_fire_11class": {"avg_ms": 46.976, "count": 114, "max_ms": 64.958, "p95_ms": 57.852}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.446, "count": 114, "max_ms": 89.616, "p95_ms": 84.979}}, "truncated_count": 0}, "processing_total": {"avg_ms": 252.719, "count": 115, "max_ms": 306.853, "p95_ms": 275.842}, "read_interval": {"avg_ms": 262.197, "count": 114, "max_ms": 316.648, "p95_ms": 286.179}, "triton_wait": {"avg_ms": 68.396, "count": 114, "max_ms": 89.627, "p95_ms": 86.583}, "visualize": {"avg_ms": 11.794, "count": 114, "max_ms": 15.349, "p95_ms": 12.904}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 30}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5854.16, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:08:50+09:00", "last_error_summary": "[h264 @ 0x555d37690b40] error while decoding MB 6 32, bytestream -7", "last_exit_at": "2026-07-01T12:08:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.174, "last_started_at": "2026-07-01T12:06:59+09:00", "pid": 3928, "process": {"exists": false, "pid": 3928}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 511.098, "sample_count": 880, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3634, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:08:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:08:50+09:00", "last_frame_read_age_sec": 0.265, "last_frame_read_at": "2026-07-01T12:08:50+09:00", "mjpeg": {"age_sec": 0.256, "exists": true, "mtime": "2026-07-01T12:08:50.632216+09:00", "mtime_epoch": 1782875330.6322157, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174401}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:08:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2346.297, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.845, "pushed": 3.845, "read": 3.811}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.964, "count": 114, "max_ms": 15.361, "p95_ms": 13.702}, "gstreamer_push": {"avg_ms": 0.862, "count": 115, "max_ms": 1.235, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.391, "count": 114, "max_ms": 48.438, "p95_ms": 43.142}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.701, "count": 114, "max_ms": 88.829, "p95_ms": 85.032}, "yolo11_640_fire_11class": {"avg_ms": 46.976, "count": 114, "max_ms": 64.958, "p95_ms": 57.852}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.446, "count": 114, "max_ms": 89.616, "p95_ms": 84.979}}, "truncated_count": 0}, "processing_total": {"avg_ms": 252.719, "count": 115, "max_ms": 306.853, "p95_ms": 275.842}, "read_interval": {"avg_ms": 262.197, "count": 114, "max_ms": 316.648, "p95_ms": 286.179}, "triton_wait": {"avg_ms": 68.396, "count": 114, "max_ms": 89.627, "p95_ms": 86.583}, "visualize": {"avg_ms": 11.794, "count": 114, "max_ms": 15.349, "p95_ms": 12.904}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 30}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5962.582, "limit_mb": 8796093022207.996}, "detail": {"pid": 4035, "restart_count": 9, "start_count": 10}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:08:50+09:00", "last_error_summary": "[h264 @ 0x555d37690b40] error while decoding MB 6 32, bytestream -7", "last_exit_at": "2026-07-01T12:08:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:08:53+09:00", "pid": 4035, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.121, "name": "ffmpeg", "pid": 4035, "rss_mb": 139.121, "state": "S (sleeping)", "threads": 106}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 514.434, "sample_count": 864, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3634, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:08:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.128, "last_frame_pushed_at": "2026-07-01T12:08:50+09:00", "last_frame_read_age_sec": 4.387, "last_frame_read_at": "2026-07-01T12:08:50+09:00", "mjpeg": {"age_sec": 4.113, "exists": true, "mtime": "2026-07-01T12:08:50.896220+09:00", "mtime_epoch": 1782875330.8962197, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174192}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:08:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2346.297, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.306, "pushed": 3.306, "read": 3.272}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.04, "count": 98, "max_ms": 15.361, "p95_ms": 14.426}, "gstreamer_push": {"avg_ms": 0.864, "count": 99, "max_ms": 1.235, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.689, "count": 98, "max_ms": 48.438, "p95_ms": 43.166}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.183, "count": 98, "max_ms": 88.829, "p95_ms": 85.782}, "yolo11_640_fire_11class": {"avg_ms": 47.678, "count": 98, "max_ms": 64.958, "p95_ms": 57.767}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.225, "count": 98, "max_ms": 89.616, "p95_ms": 86.571}}, "truncated_count": 0}, "processing_total": {"avg_ms": 253.829, "count": 99, "max_ms": 306.853, "p95_ms": 277.118}, "read_interval": {"avg_ms": 263.337, "count": 98, "max_ms": 316.648, "p95_ms": 286.57}, "triton_wait": {"avg_ms": 68.958, "count": 98, "max_ms": 89.627, "p95_ms": 86.783}, "visualize": {"avg_ms": 11.843, "count": 98, "max_ms": 15.349, "p95_ms": 13.092}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 32}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6029.359, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558a1d0bc700] error while decoding MB 56 35, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:11:02+09:00", "last_error_summary": "[h264 @ 0x558a1d0bc700] error while decoding MB 56 35, bytestream -9", "last_exit_at": "2026-07-01T12:08:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.181, "last_started_at": "2026-07-01T12:08:53+09:00", "pid": 4035, "process": {"exists": false, "pid": 4035}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 459.999, "sample_count": 852, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3953, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:11:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T12:11:02+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T12:11:02+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T12:11:02.146285+09:00", "mtime_epoch": 1782875462.1462853, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183386}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:11:02+09:00", "process": {"exists": true, "fd_count": 127, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.039, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 0.947, "recorders": [{"age_sec": 0.947, "frame_count": 51, "is_recording": true, "is_stopping": false, "pid": 4202, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.672, "name": "ffmpeg", "pid": 4202, "rss_mb": 185.672, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 49, "returncode": null, "running": true, "started_at": "2026-07-01T12:11:01+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_121101_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [4202], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 75, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.505, "pushed": 2.505, "read": 2.505}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.076, "count": 74, "max_ms": 20.989, "p95_ms": 20.507}, "gstreamer_push": {"avg_ms": 0.871, "count": 75, "max_ms": 0.94, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.099, "count": 74, "max_ms": 44.899, "p95_ms": 39.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.421, "count": 74, "max_ms": 93.949, "p95_ms": 86.662}, "yolo11_640_fire_11class": {"avg_ms": 46.3, "count": 74, "max_ms": 56.793, "p95_ms": 51.946}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.888, "count": 74, "max_ms": 90.519, "p95_ms": 85.808}}, "truncated_count": 0}, "processing_total": {"avg_ms": 394.496, "count": 75, "max_ms": 1014.379, "p95_ms": 412.694}, "read_interval": {"avg_ms": 403.954, "count": 75, "max_ms": 1024.474, "p95_ms": 421.439}, "triton_wait": {"avg_ms": 71.08, "count": 74, "max_ms": 93.963, "p95_ms": 88.381}, "visualize": {"avg_ms": 20.602, "count": 74, "max_ms": 23.998, "p95_ms": 22.062}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 32}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6028.016, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:11:02+09:00", "last_error_summary": "[h264 @ 0x558a1d0bc700] error while decoding MB 56 35, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:11:02+09:00", "last_error_summary": "[h264 @ 0x558a1d0bc700] error while decoding MB 56 35, bytestream -9", "last_exit_at": "2026-07-01T12:11:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.563, "last_started_at": "2026-07-01T12:08:53+09:00", "pid": 4035, "process": {"exists": false, "pid": 4035}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 459.999, "sample_count": 852, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3954, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:11:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:11:02+09:00", "last_frame_read_age_sec": 0.435, "last_frame_read_at": "2026-07-01T12:11:02+09:00", "mjpeg": {"age_sec": 0.426, "exists": true, "mtime": "2026-07-01T12:11:02.146285+09:00", "mtime_epoch": 1782875462.1462853, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183386}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:11:02+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.039, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 1.328, "recorders": [{"age_sec": 1.328, "frame_count": 52, "is_recording": true, "is_stopping": false, "pid": 4202, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.672, "name": "ffmpeg", "pid": 4202, "rss_mb": 185.672, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 48, "returncode": null, "running": true, "started_at": "2026-07-01T12:11:01+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_121101_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [4202], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 74, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.506, "pushed": 2.506, "read": 2.472}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.099, "count": 74, "max_ms": 22.0, "p95_ms": 20.532}, "gstreamer_push": {"avg_ms": 0.87, "count": 75, "max_ms": 0.94, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.11, "count": 74, "max_ms": 44.899, "p95_ms": 39.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.317, "count": 74, "max_ms": 93.949, "p95_ms": 86.662}, "yolo11_640_fire_11class": {"avg_ms": 46.116, "count": 74, "max_ms": 56.793, "p95_ms": 51.946}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.782, "count": 74, "max_ms": 90.519, "p95_ms": 85.808}}, "truncated_count": 0}, "processing_total": {"avg_ms": 394.836, "count": 75, "max_ms": 1014.379, "p95_ms": 419.462}, "read_interval": {"avg_ms": 403.793, "count": 74, "max_ms": 1024.474, "p95_ms": 421.439}, "triton_wait": {"avg_ms": 70.976, "count": 74, "max_ms": 93.963, "p95_ms": 88.381}, "visualize": {"avg_ms": 20.722, "count": 74, "max_ms": 29.47, "p95_ms": 22.256}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 32}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6028.008, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:11:02+09:00", "last_error_summary": "[h264 @ 0x558a1d0bc700] error while decoding MB 56 35, bytestream -9", "last_exit_at": "2026-07-01T12:11:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.567, "last_started_at": "2026-07-01T12:08:53+09:00", "pid": 4035, "process": {"exists": false, "pid": 4035}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 459.999, "sample_count": 852, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3954, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:11:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:11:02+09:00", "last_frame_read_age_sec": 0.438, "last_frame_read_at": "2026-07-01T12:11:02+09:00", "mjpeg": {"age_sec": 0.429, "exists": true, "mtime": "2026-07-01T12:11:02.146285+09:00", "mtime_epoch": 1782875462.1462853, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183386}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:11:02+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.039, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 1.331, "recorders": [{"age_sec": 1.331, "frame_count": 52, "is_recording": true, "is_stopping": false, "pid": 4202, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.672, "name": "ffmpeg", "pid": 4202, "rss_mb": 185.672, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 48, "returncode": null, "running": true, "started_at": "2026-07-01T12:11:01+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_121101_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [4202], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 74, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.505, "pushed": 2.505, "read": 2.472}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.099, "count": 74, "max_ms": 22.0, "p95_ms": 20.532}, "gstreamer_push": {"avg_ms": 0.87, "count": 75, "max_ms": 0.94, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.11, "count": 74, "max_ms": 44.899, "p95_ms": 39.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.317, "count": 74, "max_ms": 93.949, "p95_ms": 86.662}, "yolo11_640_fire_11class": {"avg_ms": 46.116, "count": 74, "max_ms": 56.793, "p95_ms": 51.946}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.782, "count": 74, "max_ms": 90.519, "p95_ms": 85.808}}, "truncated_count": 0}, "processing_total": {"avg_ms": 394.836, "count": 75, "max_ms": 1014.379, "p95_ms": 419.462}, "read_interval": {"avg_ms": 403.793, "count": 74, "max_ms": 1024.474, "p95_ms": 421.439}, "triton_wait": {"avg_ms": 70.976, "count": 74, "max_ms": 93.963, "p95_ms": 88.381}, "visualize": {"avg_ms": 20.722, "count": 74, "max_ms": 29.47, "p95_ms": 22.256}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 32}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6136.254, "limit_mb": 8796093022207.996}, "detail": {"pid": 4262, "restart_count": 10, "start_count": 11}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:11:02+09:00", "last_error_summary": "[h264 @ 0x558a1d0bc700] error while decoding MB 56 35, bytestream -9", "last_exit_at": "2026-07-01T12:11:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:11:05+09:00", "pid": 4262, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.793, "name": "ffmpeg", "pid": 4262, "rss_mb": 138.793, "state": "S (sleeping)", "threads": 106}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 461.105, "sample_count": 840, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3954, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:11:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.965, "last_frame_pushed_at": "2026-07-01T12:11:02+09:00", "last_frame_read_age_sec": 5.397, "last_frame_read_at": "2026-07-01T12:11:02+09:00", "mjpeg": {"age_sec": 4.948, "exists": true, "mtime": "2026-07-01T12:11:02.586292+09:00", "mtime_epoch": 1782875462.5862923, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183466}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:11:07+09:00", "process": {"exists": true, "fd_count": 123, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.039, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 6.292, "recorders": [{"age_sec": 6.292, "frame_count": 52, "is_recording": true, "is_stopping": false, "pid": 4202, "process": {"exists": true, "fd_count": 5, "hwm_mb": 185.672, "name": "ffmpeg", "pid": 4202, "rss_mb": 185.672, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 48, "returncode": null, "running": true, "started_at": "2026-07-01T12:11:01+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260701_121101_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [4202], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.089, "pushed": 2.089, "read": 2.055}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.069, "count": 62, "max_ms": 22.0, "p95_ms": 20.409}, "gstreamer_push": {"avg_ms": 0.87, "count": 62, "max_ms": 0.94, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.1, "count": 62, "max_ms": 44.899, "p95_ms": 39.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.184, "count": 62, "max_ms": 93.949, "p95_ms": 85.906}, "yolo11_640_fire_11class": {"avg_ms": 45.997, "count": 62, "max_ms": 56.793, "p95_ms": 51.946}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.556, "count": 62, "max_ms": 90.519, "p95_ms": 85.712}}, "truncated_count": 0}, "processing_total": {"avg_ms": 395.527, "count": 62, "max_ms": 1014.379, "p95_ms": 412.694}, "read_interval": {"avg_ms": 404.432, "count": 61, "max_ms": 1024.474, "p95_ms": 417.183}, "triton_wait": {"avg_ms": 70.908, "count": 62, "max_ms": 93.963, "p95_ms": 85.919}, "visualize": {"avg_ms": 20.754, "count": 62, "max_ms": 29.47, "p95_ms": 22.001}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.684, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5883.66, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55866260af80] error while decoding MB 79 21, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:13:13+09:00", "last_error_summary": "[h264 @ 0x55866260af80] error while decoding MB 79 21, bytestream -5", "last_exit_at": "2026-07-01T12:11:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.381, "last_started_at": "2026-07-01T12:11:05+09:00", "pid": 4262, "process": {"exists": false, "pid": 4262}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 438.919, "sample_count": 780, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:13:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T12:13:13+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T12:13:13+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T12:13:13.872384+09:00", "mtime_epoch": 1782875593.8723843, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177914}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:13:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.297, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 79, "pushed_frames": 79, "read_frames": 79, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.662, "pushed": 2.662, "read": 2.662}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.935, "count": 79, "max_ms": 23.665, "p95_ms": 23.403}, "gstreamer_push": {"avg_ms": 0.844, "count": 79, "max_ms": 0.939, "p95_ms": 0.886}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.736, "count": 79, "max_ms": 44.022, "p95_ms": 42.766}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.291, "count": 79, "max_ms": 95.154, "p95_ms": 85.162}, "yolo11_640_fire_11class": {"avg_ms": 47.418, "count": 79, "max_ms": 59.365, "p95_ms": 56.635}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.521, "count": 79, "max_ms": 94.288, "p95_ms": 84.861}}, "truncated_count": 0}, "processing_total": {"avg_ms": 370.727, "count": 79, "max_ms": 432.872, "p95_ms": 407.443}, "read_interval": {"avg_ms": 380.003, "count": 79, "max_ms": 442.068, "p95_ms": 416.458}, "triton_wait": {"avg_ms": 72.075, "count": 79, "max_ms": 95.166, "p95_ms": 86.121}, "visualize": {"avg_ms": 15.926, "count": 79, "max_ms": 21.748, "p95_ms": 20.377}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.679, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5882.836, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:13:13+09:00", "last_error_summary": "[h264 @ 0x55866260af80] error while decoding MB 79 21, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:13:13+09:00", "last_error_summary": "[h264 @ 0x55866260af80] error while decoding MB 79 21, bytestream -5", "last_exit_at": "2026-07-01T12:13:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.72, "last_started_at": "2026-07-01T12:11:05+09:00", "pid": 4262, "process": {"exists": false, "pid": 4262}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 438.919, "sample_count": 779, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4273, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:13:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T12:13:14+09:00", "last_frame_read_age_sec": 0.39, "last_frame_read_at": "2026-07-01T12:13:13+09:00", "mjpeg": {"age_sec": 0.381, "exists": true, "mtime": "2026-07-01T12:13:13.872384+09:00", "mtime_epoch": 1782875593.8723843, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177914}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:13:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.297, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 79, "pushed_frames": 79, "read_frames": 78, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.666, "pushed": 2.666, "read": 2.632}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.937, "count": 79, "max_ms": 23.665, "p95_ms": 23.403}, "gstreamer_push": {"avg_ms": 0.843, "count": 79, "max_ms": 0.939, "p95_ms": 0.886}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.751, "count": 79, "max_ms": 44.022, "p95_ms": 42.766}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.175, "count": 79, "max_ms": 95.154, "p95_ms": 85.162}, "yolo11_640_fire_11class": {"avg_ms": 47.236, "count": 79, "max_ms": 59.365, "p95_ms": 56.635}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.404, "count": 79, "max_ms": 94.288, "p95_ms": 84.861}}, "truncated_count": 0}, "processing_total": {"avg_ms": 370.635, "count": 79, "max_ms": 432.872, "p95_ms": 407.443}, "read_interval": {"avg_ms": 379.741, "count": 78, "max_ms": 442.068, "p95_ms": 416.458}, "triton_wait": {"avg_ms": 71.959, "count": 79, "max_ms": 95.166, "p95_ms": 86.121}, "visualize": {"avg_ms": 15.92, "count": 79, "max_ms": 21.748, "p95_ms": 20.377}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.632, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5882.828, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:13:13+09:00", "last_error_summary": "[h264 @ 0x55866260af80] error while decoding MB 79 21, bytestream -5", "last_exit_at": "2026-07-01T12:13:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.723, "last_started_at": "2026-07-01T12:11:05+09:00", "pid": 4262, "process": {"exists": false, "pid": 4262}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 438.919, "sample_count": 779, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4273, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:13:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T12:13:14+09:00", "last_frame_read_age_sec": 0.393, "last_frame_read_at": "2026-07-01T12:13:13+09:00", "mjpeg": {"age_sec": 0.384, "exists": true, "mtime": "2026-07-01T12:13:13.872384+09:00", "mtime_epoch": 1782875593.8723843, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177914}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:13:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.297, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 79, "pushed_frames": 79, "read_frames": 78, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.666, "pushed": 2.666, "read": 2.632}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.937, "count": 79, "max_ms": 23.665, "p95_ms": 23.403}, "gstreamer_push": {"avg_ms": 0.843, "count": 79, "max_ms": 0.939, "p95_ms": 0.886}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.751, "count": 79, "max_ms": 44.022, "p95_ms": 42.766}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.175, "count": 79, "max_ms": 95.154, "p95_ms": 85.162}, "yolo11_640_fire_11class": {"avg_ms": 47.236, "count": 79, "max_ms": 59.365, "p95_ms": 56.635}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.404, "count": 79, "max_ms": 94.288, "p95_ms": 84.861}}, "truncated_count": 0}, "processing_total": {"avg_ms": 370.635, "count": 79, "max_ms": 432.872, "p95_ms": 407.443}, "read_interval": {"avg_ms": 379.741, "count": 78, "max_ms": 442.068, "p95_ms": 416.458}, "triton_wait": {"avg_ms": 71.959, "count": 79, "max_ms": 95.166, "p95_ms": 86.121}, "visualize": {"avg_ms": 15.92, "count": 79, "max_ms": 21.748, "p95_ms": 20.377}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.635, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5991.262, "limit_mb": 8796093022207.996}, "detail": {"pid": 4369, "restart_count": 11, "start_count": 12}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:13:13+09:00", "last_error_summary": "[h264 @ 0x55866260af80] error while decoding MB 79 21, bytestream -5", "last_exit_at": "2026-07-01T12:13:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:13:16+09:00", "pid": 4369, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.086, "name": "ffmpeg", "pid": 4369, "rss_mb": 139.086, "state": "S (sleeping)", "threads": 106}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 439.553, "sample_count": 762, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4273, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:13:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.568, "last_frame_pushed_at": "2026-07-01T12:13:14+09:00", "last_frame_read_age_sec": 4.953, "last_frame_read_at": "2026-07-01T12:13:13+09:00", "mjpeg": {"age_sec": 4.553, "exists": true, "mtime": "2026-07-01T12:13:14.264391+09:00", "mtime_epoch": 1782875594.2643907, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178263}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:13:18+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.297, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.284, "pushed": 2.284, "read": 2.251}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 18.548, "count": 67, "max_ms": 23.665, "p95_ms": 23.217}, "gstreamer_push": {"avg_ms": 0.841, "count": 68, "max_ms": 0.918, "p95_ms": 0.872}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.834, "count": 67, "max_ms": 44.022, "p95_ms": 42.766}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.596, "count": 67, "max_ms": 95.154, "p95_ms": 85.162}, "yolo11_640_fire_11class": {"avg_ms": 46.88, "count": 67, "max_ms": 59.365, "p95_ms": 55.432}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.516, "count": 67, "max_ms": 94.288, "p95_ms": 84.861}}, "truncated_count": 0}, "processing_total": {"avg_ms": 366.898, "count": 68, "max_ms": 413.53, "p95_ms": 399.459}, "read_interval": {"avg_ms": 375.893, "count": 67, "max_ms": 422.202, "p95_ms": 408.288}, "triton_wait": {"avg_ms": 71.467, "count": 67, "max_ms": 95.166, "p95_ms": 86.121}, "visualize": {"avg_ms": 16.097, "count": 67, "max_ms": 21.748, "p95_ms": 20.377}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.768, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5885.324, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56261f58fe80] error while decoding MB 46 7, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:15:20+09:00", "last_error_summary": "[h264 @ 0x56261f58fe80] error while decoding MB 46 7, bytestream -5", "last_exit_at": "2026-07-01T12:13:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.571, "last_started_at": "2026-07-01T12:13:16+09:00", "pid": 4369, "process": {"exists": false, "pid": 4369}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 424.082, "sample_count": 816, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4667, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:15:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-01T12:15:20+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-01T12:15:20+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T12:15:20.350419+09:00", "mtime_epoch": 1782875720.3504188, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183519}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:15:20+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.297, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.225, "pushed": 3.225, "read": 3.225}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.462, "count": 96, "max_ms": 20.377, "p95_ms": 16.609}, "gstreamer_push": {"avg_ms": 0.821, "count": 96, "max_ms": 1.05, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.268, "count": 96, "max_ms": 44.513, "p95_ms": 39.665}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.731, "count": 96, "max_ms": 95.943, "p95_ms": 82.768}, "yolo11_640_fire_11class": {"avg_ms": 47.533, "count": 96, "max_ms": 60.595, "p95_ms": 57.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.32, "count": 96, "max_ms": 94.328, "p95_ms": 87.723}}, "truncated_count": 0}, "processing_total": {"avg_ms": 303.316, "count": 96, "max_ms": 351.256, "p95_ms": 338.208}, "read_interval": {"avg_ms": 312.622, "count": 96, "max_ms": 360.7, "p95_ms": 347.863}, "triton_wait": {"avg_ms": 70.653, "count": 96, "max_ms": 95.955, "p95_ms": 87.74}, "visualize": {"avg_ms": 14.027, "count": 96, "max_ms": 20.613, "p95_ms": 16.872}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.766, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5884.457, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:15:20+09:00", "last_error_summary": "[h264 @ 0x56261f58fe80] error while decoding MB 46 7, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:15:20+09:00", "last_error_summary": "[h264 @ 0x56261f58fe80] error while decoding MB 46 7, bytestream -5", "last_exit_at": "2026-07-01T12:15:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.838, "last_started_at": "2026-07-01T12:13:16+09:00", "pid": 4369, "process": {"exists": false, "pid": 4369}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 424.082, "sample_count": 817, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:15:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:15:20+09:00", "last_frame_read_age_sec": 0.315, "last_frame_read_at": "2026-07-01T12:15:20+09:00", "mjpeg": {"age_sec": 0.304, "exists": true, "mtime": "2026-07-01T12:15:20.350419+09:00", "mtime_epoch": 1782875720.3504188, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183519}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:15:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.297, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.229, "pushed": 3.229, "read": 3.196}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.477, "count": 96, "max_ms": 20.377, "p95_ms": 16.609}, "gstreamer_push": {"avg_ms": 0.823, "count": 96, "max_ms": 1.05, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.286, "count": 96, "max_ms": 44.513, "p95_ms": 39.665}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.776, "count": 96, "max_ms": 95.943, "p95_ms": 82.768}, "yolo11_640_fire_11class": {"avg_ms": 47.541, "count": 96, "max_ms": 60.595, "p95_ms": 57.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.398, "count": 96, "max_ms": 94.328, "p95_ms": 87.723}}, "truncated_count": 0}, "processing_total": {"avg_ms": 303.487, "count": 96, "max_ms": 351.256, "p95_ms": 338.208}, "read_interval": {"avg_ms": 312.729, "count": 95, "max_ms": 360.7, "p95_ms": 347.863}, "triton_wait": {"avg_ms": 70.698, "count": 96, "max_ms": 95.955, "p95_ms": 87.74}, "visualize": {"avg_ms": 14.006, "count": 96, "max_ms": 20.613, "p95_ms": 16.872}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.726, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5884.449, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:15:20+09:00", "last_error_summary": "[h264 @ 0x56261f58fe80] error while decoding MB 46 7, bytestream -5", "last_exit_at": "2026-07-01T12:15:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.841, "last_started_at": "2026-07-01T12:13:16+09:00", "pid": 4369, "process": {"exists": false, "pid": 4369}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 424.082, "sample_count": 817, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:15:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:15:20+09:00", "last_frame_read_age_sec": 0.318, "last_frame_read_at": "2026-07-01T12:15:20+09:00", "mjpeg": {"age_sec": 0.307, "exists": true, "mtime": "2026-07-01T12:15:20.350419+09:00", "mtime_epoch": 1782875720.3504188, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183519}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:15:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.297, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.229, "pushed": 3.229, "read": 3.196}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.477, "count": 96, "max_ms": 20.377, "p95_ms": 16.609}, "gstreamer_push": {"avg_ms": 0.823, "count": 96, "max_ms": 1.05, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.286, "count": 96, "max_ms": 44.513, "p95_ms": 39.665}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.776, "count": 96, "max_ms": 95.943, "p95_ms": 82.768}, "yolo11_640_fire_11class": {"avg_ms": 47.541, "count": 96, "max_ms": 60.595, "p95_ms": 57.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.398, "count": 96, "max_ms": 94.328, "p95_ms": 87.723}}, "truncated_count": 0}, "processing_total": {"avg_ms": 303.487, "count": 96, "max_ms": 351.256, "p95_ms": 338.208}, "read_interval": {"avg_ms": 312.729, "count": 95, "max_ms": 360.7, "p95_ms": 347.863}, "triton_wait": {"avg_ms": 70.698, "count": 96, "max_ms": 95.955, "p95_ms": 87.74}, "visualize": {"avg_ms": 14.006, "count": 96, "max_ms": 20.613, "p95_ms": 16.872}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.729, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 38}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5999.973, "limit_mb": 8796093022207.996}, "detail": {"pid": 4476, "restart_count": 12, "start_count": 13}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:15:20+09:00", "last_error_summary": "[h264 @ 0x56261f58fe80] error while decoding MB 46 7, bytestream -5", "last_exit_at": "2026-07-01T12:15:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:15:23+09:00", "pid": 4476, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.453, "name": "ffmpeg", "pid": 4476, "rss_mb": 147.453, "state": "S (sleeping)", "threads": 106}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 422.786, "sample_count": 806, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:15:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.759, "last_frame_pushed_at": "2026-07-01T12:15:20+09:00", "last_frame_read_age_sec": 5.071, "last_frame_read_at": "2026-07-01T12:15:20+09:00", "mjpeg": {"age_sec": 4.74, "exists": true, "mtime": "2026-07-01T12:15:20.670424+09:00", "mtime_epoch": 1782875720.670424, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176705}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:15:25+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.297, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.706, "pushed": 2.706, "read": 2.673}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.573, "count": 80, "max_ms": 20.377, "p95_ms": 16.167}, "gstreamer_push": {"avg_ms": 0.838, "count": 81, "max_ms": 1.05, "p95_ms": 0.89}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.413, "count": 80, "max_ms": 44.513, "p95_ms": 39.569}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.232, "count": 80, "max_ms": 95.943, "p95_ms": 80.785}, "yolo11_640_fire_11class": {"avg_ms": 47.863, "count": 80, "max_ms": 60.595, "p95_ms": 57.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.282, "count": 80, "max_ms": 94.328, "p95_ms": 87.723}}, "truncated_count": 0}, "processing_total": {"avg_ms": 305.098, "count": 81, "max_ms": 351.256, "p95_ms": 338.208}, "read_interval": {"avg_ms": 314.45, "count": 80, "max_ms": 360.7, "p95_ms": 347.863}, "triton_wait": {"avg_ms": 71.336, "count": 80, "max_ms": 95.955, "p95_ms": 87.74}, "visualize": {"avg_ms": 14.057, "count": 80, "max_ms": 20.613, "p95_ms": 16.872}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 42}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5886.133, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ade4f59e40] error while decoding MB 2 32, bytestream -26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:17:14+09:00", "last_error_summary": "[h264 @ 0x55ade4f59e40] error while decoding MB 2 32, bytestream -26", "last_exit_at": "2026-07-01T12:15:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.67, "last_started_at": "2026-07-01T12:15:23+09:00", "pid": 4476, "process": {"exists": false, "pid": 4476}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 405.295, "sample_count": 964, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5086, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:17:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T12:17:14+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T12:17:14+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T12:17:14.032260+09:00", "mtime_epoch": 1782875834.0322602, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185118}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:17:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.555, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.98, "pushed": 3.98, "read": 3.98}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.525, "count": 118, "max_ms": 14.634, "p95_ms": 14.402}, "gstreamer_push": {"avg_ms": 0.861, "count": 119, "max_ms": 1.271, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.6, "count": 118, "max_ms": 46.711, "p95_ms": 42.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.841, "count": 118, "max_ms": 80.389, "p95_ms": 77.727}, "yolo11_640_fire_11class": {"avg_ms": 46.298, "count": 118, "max_ms": 63.049, "p95_ms": 58.777}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.396, "count": 118, "max_ms": 81.835, "p95_ms": 76.828}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.974, "count": 119, "max_ms": 298.164, "p95_ms": 263.871}, "read_interval": {"avg_ms": 252.947, "count": 119, "max_ms": 310.286, "p95_ms": 273.057}, "triton_wait": {"avg_ms": 65.423, "count": 118, "max_ms": 81.849, "p95_ms": 78.03}, "visualize": {"avg_ms": 11.584, "count": 118, "max_ms": 15.502, "p95_ms": 14.693}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 42}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5885.375, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:17:14+09:00", "last_error_summary": "[h264 @ 0x55ade4f59e40] error while decoding MB 2 32, bytestream -26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:17:14+09:00", "last_error_summary": "[h264 @ 0x55ade4f59e40] error while decoding MB 2 32, bytestream -26", "last_exit_at": "2026-07-01T12:17:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.869, "last_started_at": "2026-07-01T12:15:23+09:00", "pid": 4476, "process": {"exists": false, "pid": 4476}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 405.295, "sample_count": 964, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5087, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:17:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:17:14+09:00", "last_frame_read_age_sec": 0.249, "last_frame_read_at": "2026-07-01T12:17:14+09:00", "mjpeg": {"age_sec": 0.246, "exists": true, "mtime": "2026-07-01T12:17:14.032260+09:00", "mtime_epoch": 1782875834.0322602, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185118}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:17:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.555, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.985, "pushed": 3.985, "read": 3.951}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.532, "count": 118, "max_ms": 14.634, "p95_ms": 14.402}, "gstreamer_push": {"avg_ms": 0.861, "count": 119, "max_ms": 1.271, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.549, "count": 118, "max_ms": 46.711, "p95_ms": 42.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.806, "count": 118, "max_ms": 80.389, "p95_ms": 77.727}, "yolo11_640_fire_11class": {"avg_ms": 46.22, "count": 118, "max_ms": 63.049, "p95_ms": 58.777}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.353, "count": 118, "max_ms": 81.835, "p95_ms": 76.828}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.896, "count": 119, "max_ms": 298.164, "p95_ms": 263.871}, "read_interval": {"avg_ms": 252.858, "count": 118, "max_ms": 310.286, "p95_ms": 273.057}, "triton_wait": {"avg_ms": 65.388, "count": 118, "max_ms": 81.849, "p95_ms": 78.03}, "visualize": {"avg_ms": 11.59, "count": 118, "max_ms": 15.502, "p95_ms": 14.693}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 42}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5885.367, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:17:14+09:00", "last_error_summary": "[h264 @ 0x55ade4f59e40] error while decoding MB 2 32, bytestream -26", "last_exit_at": "2026-07-01T12:17:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.872, "last_started_at": "2026-07-01T12:15:23+09:00", "pid": 4476, "process": {"exists": false, "pid": 4476}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 405.295, "sample_count": 964, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5087, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:17:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:17:14+09:00", "last_frame_read_age_sec": 0.253, "last_frame_read_at": "2026-07-01T12:17:14+09:00", "mjpeg": {"age_sec": 0.249, "exists": true, "mtime": "2026-07-01T12:17:14.032260+09:00", "mtime_epoch": 1782875834.0322602, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185118}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:17:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.555, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.984, "pushed": 3.984, "read": 3.951}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.532, "count": 118, "max_ms": 14.634, "p95_ms": 14.402}, "gstreamer_push": {"avg_ms": 0.861, "count": 119, "max_ms": 1.271, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.549, "count": 118, "max_ms": 46.711, "p95_ms": 42.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.806, "count": 118, "max_ms": 80.389, "p95_ms": 77.727}, "yolo11_640_fire_11class": {"avg_ms": 46.22, "count": 118, "max_ms": 63.049, "p95_ms": 58.777}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.353, "count": 118, "max_ms": 81.835, "p95_ms": 76.828}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.896, "count": 119, "max_ms": 298.164, "p95_ms": 263.871}, "read_interval": {"avg_ms": 252.858, "count": 118, "max_ms": 310.286, "p95_ms": 273.057}, "triton_wait": {"avg_ms": 65.388, "count": 118, "max_ms": 81.849, "p95_ms": 78.03}, "visualize": {"avg_ms": 11.59, "count": 118, "max_ms": 15.502, "p95_ms": 14.693}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 42}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6001.414, "limit_mb": 8796093022207.996}, "detail": {"pid": 4583, "restart_count": 13, "start_count": 14}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:17:14+09:00", "last_error_summary": "[h264 @ 0x55ade4f59e40] error while decoding MB 2 32, bytestream -26", "last_exit_at": "2026-07-01T12:17:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:17:16+09:00", "pid": 4583, "process": {"exists": true, "fd_count": 5, "hwm_mb": 146.895, "name": "ffmpeg", "pid": 4583, "rss_mb": 146.895, "state": "S (sleeping)", "threads": 106}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 399.459, "sample_count": 955, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5087, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:17:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.212, "last_frame_pushed_at": "2026-07-01T12:17:14+09:00", "last_frame_read_age_sec": 4.458, "last_frame_read_at": "2026-07-01T12:17:14+09:00", "mjpeg": {"age_sec": 4.195, "exists": true, "mtime": "2026-07-01T12:17:14.292264+09:00", "mtime_epoch": 1782875834.2922645, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185255}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:17:18+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.555, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.434, "pushed": 3.434, "read": 3.4}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.564, "count": 102, "max_ms": 14.634, "p95_ms": 14.402}, "gstreamer_push": {"avg_ms": 0.863, "count": 103, "max_ms": 1.271, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.413, "count": 102, "max_ms": 46.711, "p95_ms": 41.981}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.534, "count": 102, "max_ms": 80.389, "p95_ms": 77.727}, "yolo11_640_fire_11class": {"avg_ms": 45.869, "count": 102, "max_ms": 63.049, "p95_ms": 58.731}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.974, "count": 102, "max_ms": 79.136, "p95_ms": 76.783}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.893, "count": 103, "max_ms": 298.164, "p95_ms": 263.871}, "read_interval": {"avg_ms": 252.773, "count": 102, "max_ms": 310.286, "p95_ms": 273.057}, "triton_wait": {"avg_ms": 65.105, "count": 102, "max_ms": 80.401, "p95_ms": 77.941}, "visualize": {"avg_ms": 11.601, "count": 102, "max_ms": 15.502, "p95_ms": 14.693}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 46}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5886.949, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55cbc1f4e680] error while decoding MB 23 44, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:19:03+09:00", "last_error_summary": "[h264 @ 0x55cbc1f4e680] error while decoding MB 23 44, bytestream -7", "last_exit_at": "2026-07-01T12:17:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 106.989, "last_started_at": "2026-07-01T12:17:16+09:00", "pid": 4583, "process": {"exists": false, "pid": 4583}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 304.286, "sample_count": 1117, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5532, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:19:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-01T12:19:03+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T12:19:03+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T12:19:03.438041+09:00", "mtime_epoch": 1782875943.4380412, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180399}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:19:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.812, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.263, "pushed": 4.263, "read": 4.263}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.097, "count": 127, "max_ms": 13.054, "p95_ms": 10.773}, "gstreamer_push": {"avg_ms": 0.848, "count": 127, "max_ms": 1.27, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.382, "count": 127, "max_ms": 48.063, "p95_ms": 43.084}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.602, "count": 127, "max_ms": 77.123, "p95_ms": 75.348}, "yolo11_640_fire_11class": {"avg_ms": 47.53, "count": 127, "max_ms": 61.889, "p95_ms": 57.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.935, "count": 127, "max_ms": 81.44, "p95_ms": 76.158}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.634, "count": 127, "max_ms": 245.809, "p95_ms": 239.577}, "read_interval": {"avg_ms": 236.048, "count": 127, "max_ms": 256.408, "p95_ms": 248.675}, "triton_wait": {"avg_ms": 64.786, "count": 127, "max_ms": 81.453, "p95_ms": 76.88}, "visualize": {"avg_ms": 10.619, "count": 127, "max_ms": 15.254, "p95_ms": 11.301}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.792, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 46}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5886.051, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:19:03+09:00", "last_error_summary": "[h264 @ 0x55cbc1f4e680] error while decoding MB 23 44, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:19:03+09:00", "last_error_summary": "[h264 @ 0x55cbc1f4e680] error while decoding MB 23 44, bytestream -7", "last_exit_at": "2026-07-01T12:19:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.171, "last_started_at": "2026-07-01T12:17:16+09:00", "pid": 4583, "process": {"exists": false, "pid": 4583}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 304.286, "sample_count": 1118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5533, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:19:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T12:19:03+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T12:19:03+09:00", "mjpeg": {"age_sec": 0.219, "exists": true, "mtime": "2026-07-01T12:19:03.438041+09:00", "mtime_epoch": 1782875943.4380412, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180399}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:19:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.812, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.27, "pushed": 4.27, "read": 4.237}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.097, "count": 127, "max_ms": 13.054, "p95_ms": 10.773}, "gstreamer_push": {"avg_ms": 0.848, "count": 128, "max_ms": 1.27, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.333, "count": 127, "max_ms": 48.063, "p95_ms": 43.084}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.495, "count": 127, "max_ms": 77.123, "p95_ms": 75.348}, "yolo11_640_fire_11class": {"avg_ms": 47.423, "count": 127, "max_ms": 61.889, "p95_ms": 57.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.847, "count": 127, "max_ms": 81.44, "p95_ms": 76.158}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.619, "count": 128, "max_ms": 245.809, "p95_ms": 239.577}, "read_interval": {"avg_ms": 236.048, "count": 127, "max_ms": 256.408, "p95_ms": 248.675}, "triton_wait": {"avg_ms": 64.698, "count": 127, "max_ms": 81.453, "p95_ms": 76.88}, "visualize": {"avg_ms": 10.621, "count": 127, "max_ms": 15.254, "p95_ms": 11.301}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 46}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5886.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:19:03+09:00", "last_error_summary": "[h264 @ 0x55cbc1f4e680] error while decoding MB 23 44, bytestream -7", "last_exit_at": "2026-07-01T12:19:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.174, "last_started_at": "2026-07-01T12:17:16+09:00", "pid": 4583, "process": {"exists": false, "pid": 4583}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 304.286, "sample_count": 1118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5533, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:19:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T12:19:03+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-01T12:19:03+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T12:19:03.438041+09:00", "mtime_epoch": 1782875943.4380412, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180399}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:19:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.812, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.27, "pushed": 4.27, "read": 4.237}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.097, "count": 127, "max_ms": 13.054, "p95_ms": 10.773}, "gstreamer_push": {"avg_ms": 0.848, "count": 128, "max_ms": 1.27, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.333, "count": 127, "max_ms": 48.063, "p95_ms": 43.084}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.495, "count": 127, "max_ms": 77.123, "p95_ms": 75.348}, "yolo11_640_fire_11class": {"avg_ms": 47.423, "count": 127, "max_ms": 61.889, "p95_ms": 57.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.847, "count": 127, "max_ms": 81.44, "p95_ms": 76.158}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.619, "count": 128, "max_ms": 245.809, "p95_ms": 239.577}, "read_interval": {"avg_ms": 236.048, "count": 127, "max_ms": 256.408, "p95_ms": 248.675}, "triton_wait": {"avg_ms": 64.698, "count": 127, "max_ms": 81.453, "p95_ms": 76.88}, "visualize": {"avg_ms": 10.621, "count": 127, "max_ms": 15.254, "p95_ms": 11.301}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 46}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5993.645, "limit_mb": 8796093022207.996}, "detail": {"pid": 4690, "restart_count": 14, "start_count": 15}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:19:03+09:00", "last_error_summary": "[h264 @ 0x55cbc1f4e680] error while decoding MB 23 44, bytestream -7", "last_exit_at": "2026-07-01T12:19:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:19:05+09:00", "pid": 4690, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.23, "name": "ffmpeg", "pid": 4690, "rss_mb": 139.23, "state": "S (sleeping)", "threads": 106}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 304.781, "sample_count": 1104, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5533, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:19:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.124, "last_frame_pushed_at": "2026-07-01T12:19:03+09:00", "last_frame_read_age_sec": 4.35, "last_frame_read_at": "2026-07-01T12:19:03+09:00", "mjpeg": {"age_sec": 4.107, "exists": true, "mtime": "2026-07-01T12:19:03.670045+09:00", "mtime_epoch": 1782875943.670045, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180705}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:19:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2347.812, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.686, "pushed": 3.686, "read": 3.652}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.135, "count": 110, "max_ms": 13.054, "p95_ms": 10.786}, "gstreamer_push": {"avg_ms": 0.848, "count": 110, "max_ms": 1.27, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.197, "count": 110, "max_ms": 48.063, "p95_ms": 43.807}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.491, "count": 110, "max_ms": 76.869, "p95_ms": 75.348}, "yolo11_640_fire_11class": {"avg_ms": 47.41, "count": 110, "max_ms": 61.889, "p95_ms": 57.957}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.644, "count": 110, "max_ms": 81.44, "p95_ms": 75.686}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.438, "count": 110, "max_ms": 245.809, "p95_ms": 238.079}, "read_interval": {"avg_ms": 235.864, "count": 109, "max_ms": 256.408, "p95_ms": 247.868}, "triton_wait": {"avg_ms": 64.589, "count": 110, "max_ms": 81.453, "p95_ms": 76.179}, "visualize": {"avg_ms": 10.621, "count": 110, "max_ms": 15.254, "p95_ms": 11.323}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.843, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 50}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5888.059, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b7c03a8d40] error while decoding MB 79 36, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:21:12+09:00", "last_error_summary": "[h264 @ 0x55b7c03a8d40] error while decoding MB 79 36, bytestream -5", "last_exit_at": "2026-07-01T12:19:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.477, "last_started_at": "2026-07-01T12:19:05+09:00", "pid": 4690, "process": {"exists": false, "pid": 4690}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.739, "sample_count": 1265, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6107, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:21:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T12:21:12+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T12:21:12+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T12:21:12.204146+09:00", "mtime_epoch": 1782876072.2041461, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173230}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:21:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2348.07, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.706, "pushed": 4.706, "read": 4.706}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.261, "count": 140, "max_ms": 10.893, "p95_ms": 10.388}, "gstreamer_push": {"avg_ms": 0.847, "count": 141, "max_ms": 0.891, "p95_ms": 0.881}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.732, "count": 140, "max_ms": 49.667, "p95_ms": 40.537}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.003, "count": 140, "max_ms": 75.93, "p95_ms": 70.513}, "yolo11_640_fire_11class": {"avg_ms": 46.027, "count": 140, "max_ms": 61.232, "p95_ms": 54.829}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.694, "count": 140, "max_ms": 77.322, "p95_ms": 72.487}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.044, "count": 141, "max_ms": 234.712, "p95_ms": 221.713}, "read_interval": {"avg_ms": 213.555, "count": 141, "max_ms": 243.871, "p95_ms": 230.846}, "triton_wait": {"avg_ms": 60.004, "count": 140, "max_ms": 77.334, "p95_ms": 72.513}, "visualize": {"avg_ms": 9.702, "count": 140, "max_ms": 11.033, "p95_ms": 10.481}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 50}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5887.004, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:21:12+09:00", "last_error_summary": "[h264 @ 0x55b7c03a8d40] error while decoding MB 79 36, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:21:12+09:00", "last_error_summary": "[h264 @ 0x55b7c03a8d40] error while decoding MB 79 36, bytestream -5", "last_exit_at": "2026-07-01T12:21:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.633, "last_started_at": "2026-07-01T12:19:05+09:00", "pid": 4690, "process": {"exists": false, "pid": 4690}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.739, "sample_count": 1266, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6108, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:21:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:21:12+09:00", "last_frame_read_age_sec": 0.209, "last_frame_read_at": "2026-07-01T12:21:12+09:00", "mjpeg": {"age_sec": 0.205, "exists": true, "mtime": "2026-07-01T12:21:12.204146+09:00", "mtime_epoch": 1782876072.2041461, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173230}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:21:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2348.07, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.715, "pushed": 4.715, "read": 4.681}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.269, "count": 140, "max_ms": 10.893, "p95_ms": 10.401}, "gstreamer_push": {"avg_ms": 0.847, "count": 141, "max_ms": 0.891, "p95_ms": 0.881}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.74, "count": 140, "max_ms": 49.667, "p95_ms": 40.537}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.029, "count": 140, "max_ms": 75.93, "p95_ms": 70.513}, "yolo11_640_fire_11class": {"avg_ms": 46.081, "count": 140, "max_ms": 61.232, "p95_ms": 54.829}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.733, "count": 140, "max_ms": 77.322, "p95_ms": 72.487}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.049, "count": 141, "max_ms": 234.712, "p95_ms": 221.713}, "read_interval": {"avg_ms": 213.551, "count": 140, "max_ms": 243.871, "p95_ms": 230.846}, "triton_wait": {"avg_ms": 60.042, "count": 140, "max_ms": 77.334, "p95_ms": 72.513}, "visualize": {"avg_ms": 9.699, "count": 140, "max_ms": 11.033, "p95_ms": 10.481}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 50}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5886.996, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:21:12+09:00", "last_error_summary": "[h264 @ 0x55b7c03a8d40] error while decoding MB 79 36, bytestream -5", "last_exit_at": "2026-07-01T12:21:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.637, "last_started_at": "2026-07-01T12:19:05+09:00", "pid": 4690, "process": {"exists": false, "pid": 4690}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.548, "sample_count": 1265, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6108, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:21:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T12:21:12+09:00", "last_frame_read_age_sec": 0.213, "last_frame_read_at": "2026-07-01T12:21:12+09:00", "mjpeg": {"age_sec": 0.208, "exists": true, "mtime": "2026-07-01T12:21:12.204146+09:00", "mtime_epoch": 1782876072.2041461, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173230}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:21:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2348.07, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.714, "pushed": 4.714, "read": 4.681}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.269, "count": 140, "max_ms": 10.893, "p95_ms": 10.401}, "gstreamer_push": {"avg_ms": 0.847, "count": 141, "max_ms": 0.891, "p95_ms": 0.881}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.74, "count": 140, "max_ms": 49.667, "p95_ms": 40.537}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.029, "count": 140, "max_ms": 75.93, "p95_ms": 70.513}, "yolo11_640_fire_11class": {"avg_ms": 46.081, "count": 140, "max_ms": 61.232, "p95_ms": 54.829}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.733, "count": 140, "max_ms": 77.322, "p95_ms": 72.487}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.049, "count": 141, "max_ms": 234.712, "p95_ms": 221.713}, "read_interval": {"avg_ms": 213.551, "count": 140, "max_ms": 243.871, "p95_ms": 230.846}, "triton_wait": {"avg_ms": 60.042, "count": 140, "max_ms": 77.334, "p95_ms": 72.513}, "visualize": {"avg_ms": 9.699, "count": 140, "max_ms": 11.033, "p95_ms": 10.481}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 50}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5994.332, "limit_mb": 8796093022207.996}, "detail": {"pid": 4797, "restart_count": 15, "start_count": 16}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:21:12+09:00", "last_error_summary": "[h264 @ 0x55b7c03a8d40] error while decoding MB 79 36, bytestream -5", "last_exit_at": "2026-07-01T12:21:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:21:14+09:00", "pid": 4797, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.73, "name": "ffmpeg", "pid": 4797, "rss_mb": 138.73, "state": "S (sleeping)", "threads": 106}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.958, "sample_count": 1248, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6108, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:21:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.457, "last_frame_pushed_at": "2026-07-01T12:21:12+09:00", "last_frame_read_age_sec": 4.663, "last_frame_read_at": "2026-07-01T12:21:12+09:00", "mjpeg": {"age_sec": 4.443, "exists": true, "mtime": "2026-07-01T12:21:12.420150+09:00", "mtime_epoch": 1782876072.4201496, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173626}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:21:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2348.07, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.018, "pushed": 4.018, "read": 3.984}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.232, "count": 119, "max_ms": 10.893, "p95_ms": 10.457}, "gstreamer_push": {"avg_ms": 0.846, "count": 120, "max_ms": 0.891, "p95_ms": 0.881}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.746, "count": 119, "max_ms": 49.667, "p95_ms": 41.605}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.205, "count": 119, "max_ms": 75.93, "p95_ms": 71.197}, "yolo11_640_fire_11class": {"avg_ms": 45.786, "count": 119, "max_ms": 61.232, "p95_ms": 54.829}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.621, "count": 119, "max_ms": 77.322, "p95_ms": 72.503}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.182, "count": 120, "max_ms": 234.712, "p95_ms": 221.713}, "read_interval": {"avg_ms": 213.688, "count": 119, "max_ms": 243.871, "p95_ms": 231.291}, "triton_wait": {"avg_ms": 59.983, "count": 119, "max_ms": 77.334, "p95_ms": 72.875}, "visualize": {"avg_ms": 9.691, "count": 119, "max_ms": 11.033, "p95_ms": 10.481}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 55}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5894.629, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f8d0e43a80] error while decoding MB 45 9, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:23:23+09:00", "last_error_summary": "[h264 @ 0x55f8d0e43a80] error while decoding MB 45 9, bytestream -5", "last_exit_at": "2026-07-01T12:21:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.186, "last_started_at": "2026-07-01T12:21:14+09:00", "pid": 4797, "process": {"exists": false, "pid": 4797}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 233.733, "sample_count": 1338, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6698, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:23:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T12:23:22+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T12:23:22+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T12:23:23.002292+09:00", "mtime_epoch": 1782876203.0022922, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178314}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:23:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2353.484, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.734, "pushed": 4.734, "read": 4.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.168, "count": 141, "max_ms": 12.298, "p95_ms": 10.182}, "gstreamer_push": {"avg_ms": 0.854, "count": 142, "max_ms": 1.091, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.092, "count": 141, "max_ms": 53.462, "p95_ms": 41.679}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.597, "count": 141, "max_ms": 76.175, "p95_ms": 71.888}, "yolo11_640_fire_11class": {"avg_ms": 46.208, "count": 141, "max_ms": 59.753, "p95_ms": 56.418}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.944, "count": 141, "max_ms": 80.681, "p95_ms": 74.649}}, "truncated_count": 0}, "processing_total": {"avg_ms": 202.645, "count": 142, "max_ms": 238.593, "p95_ms": 217.409}, "read_interval": {"avg_ms": 212.381, "count": 142, "max_ms": 248.923, "p95_ms": 226.976}, "triton_wait": {"avg_ms": 61.374, "count": 141, "max_ms": 80.693, "p95_ms": 74.953}, "visualize": {"avg_ms": 9.571, "count": 141, "max_ms": 12.042, "p95_ms": 10.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 55}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5893.812, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:23:23+09:00", "last_error_summary": "[h264 @ 0x55f8d0e43a80] error while decoding MB 45 9, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:23:23+09:00", "last_error_summary": "[h264 @ 0x55f8d0e43a80] error while decoding MB 45 9, bytestream -5", "last_exit_at": "2026-07-01T12:23:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.341, "last_started_at": "2026-07-01T12:21:14+09:00", "pid": 4797, "process": {"exists": false, "pid": 4797}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 233.733, "sample_count": 1338, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6699, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:23:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:23:23+09:00", "last_frame_read_age_sec": 0.208, "last_frame_read_at": "2026-07-01T12:23:22+09:00", "mjpeg": {"age_sec": 0.201, "exists": true, "mtime": "2026-07-01T12:23:23.002292+09:00", "mtime_epoch": 1782876203.0022922, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178314}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:23:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2353.484, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.743, "pushed": 4.743, "read": 4.71}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.175, "count": 141, "max_ms": 12.298, "p95_ms": 10.253}, "gstreamer_push": {"avg_ms": 0.854, "count": 142, "max_ms": 1.091, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.106, "count": 141, "max_ms": 53.462, "p95_ms": 41.679}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.597, "count": 141, "max_ms": 76.175, "p95_ms": 71.888}, "yolo11_640_fire_11class": {"avg_ms": 46.232, "count": 141, "max_ms": 59.753, "p95_ms": 56.418}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.958, "count": 141, "max_ms": 80.681, "p95_ms": 74.649}}, "truncated_count": 0}, "processing_total": {"avg_ms": 202.561, "count": 142, "max_ms": 238.593, "p95_ms": 217.409}, "read_interval": {"avg_ms": 212.292, "count": 141, "max_ms": 248.923, "p95_ms": 226.976}, "triton_wait": {"avg_ms": 61.382, "count": 141, "max_ms": 80.693, "p95_ms": 74.953}, "visualize": {"avg_ms": 9.57, "count": 141, "max_ms": 12.042, "p95_ms": 10.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 55}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5893.805, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:23:23+09:00", "last_error_summary": "[h264 @ 0x55f8d0e43a80] error while decoding MB 45 9, bytestream -5", "last_exit_at": "2026-07-01T12:23:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.345, "last_started_at": "2026-07-01T12:21:14+09:00", "pid": 4797, "process": {"exists": false, "pid": 4797}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 233.733, "sample_count": 1338, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6699, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:23:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T12:23:23+09:00", "last_frame_read_age_sec": 0.212, "last_frame_read_at": "2026-07-01T12:23:22+09:00", "mjpeg": {"age_sec": 0.205, "exists": true, "mtime": "2026-07-01T12:23:23.002292+09:00", "mtime_epoch": 1782876203.0022922, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178314}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:23:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2353.484, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.743, "pushed": 4.743, "read": 4.709}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.175, "count": 141, "max_ms": 12.298, "p95_ms": 10.253}, "gstreamer_push": {"avg_ms": 0.854, "count": 142, "max_ms": 1.091, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.106, "count": 141, "max_ms": 53.462, "p95_ms": 41.679}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.597, "count": 141, "max_ms": 76.175, "p95_ms": 71.888}, "yolo11_640_fire_11class": {"avg_ms": 46.232, "count": 141, "max_ms": 59.753, "p95_ms": 56.418}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.958, "count": 141, "max_ms": 80.681, "p95_ms": 74.649}}, "truncated_count": 0}, "processing_total": {"avg_ms": 202.561, "count": 142, "max_ms": 238.593, "p95_ms": 217.409}, "read_interval": {"avg_ms": 212.292, "count": 141, "max_ms": 248.923, "p95_ms": 226.976}, "triton_wait": {"avg_ms": 61.382, "count": 141, "max_ms": 80.693, "p95_ms": 74.953}, "visualize": {"avg_ms": 9.57, "count": 141, "max_ms": 12.042, "p95_ms": 10.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 55}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6001.406, "limit_mb": 8796093022207.996}, "detail": {"pid": 4904, "restart_count": 16, "start_count": 17}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:23:23+09:00", "last_error_summary": "[h264 @ 0x55f8d0e43a80] error while decoding MB 45 9, bytestream -5", "last_exit_at": "2026-07-01T12:23:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:23:25+09:00", "pid": 4904, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.137, "name": "ffmpeg", "pid": 4904, "rss_mb": 139.137, "state": "S (sleeping)", "threads": 106}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 233.551, "sample_count": 1319, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6699, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:23:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.51, "last_frame_pushed_at": "2026-07-01T12:23:23+09:00", "last_frame_read_age_sec": 4.715, "last_frame_read_at": "2026-07-01T12:23:22+09:00", "mjpeg": {"age_sec": 4.492, "exists": true, "mtime": "2026-07-01T12:23:23.218296+09:00", "mtime_epoch": 1782876203.2182956, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178367}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:23:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2353.484, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.047, "pushed": 4.047, "read": 4.014}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.031, "count": 120, "max_ms": 12.123, "p95_ms": 9.584}, "gstreamer_push": {"avg_ms": 0.852, "count": 121, "max_ms": 1.091, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.07, "count": 120, "max_ms": 48.496, "p95_ms": 40.616}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.592, "count": 120, "max_ms": 76.175, "p95_ms": 72.152}, "yolo11_640_fire_11class": {"avg_ms": 47.187, "count": 120, "max_ms": 59.753, "p95_ms": 56.82}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.154, "count": 120, "max_ms": 80.681, "p95_ms": 74.649}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.885, "count": 121, "max_ms": 238.593, "p95_ms": 217.409}, "read_interval": {"avg_ms": 211.607, "count": 120, "max_ms": 248.923, "p95_ms": 226.976}, "triton_wait": {"avg_ms": 61.505, "count": 120, "max_ms": 80.693, "p95_ms": 74.953}, "visualize": {"avg_ms": 9.513, "count": 120, "max_ms": 11.602, "p95_ms": 10.147}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 60}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5894.957, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558848702c80] error while decoding MB 41 39, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:25:33+09:00", "last_error_summary": "[h264 @ 0x558848702c80] error while decoding MB 41 39, bytestream -5", "last_exit_at": "2026-07-01T12:23:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.427, "last_started_at": "2026-07-01T12:23:25+09:00", "pid": 4904, "process": {"exists": false, "pid": 4904}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.946, "sample_count": 1366, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7290, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:25:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T12:25:33+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T12:25:33+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T12:25:33.088432+09:00", "mtime_epoch": 1782876333.0884323, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178231}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:25:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2353.742, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.719, "pushed": 4.719, "read": 4.719}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.257, "count": 140, "max_ms": 14.56, "p95_ms": 10.146}, "gstreamer_push": {"avg_ms": 0.845, "count": 141, "max_ms": 0.948, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.399, "count": 140, "max_ms": 43.845, "p95_ms": 38.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.547, "count": 140, "max_ms": 78.311, "p95_ms": 68.141}, "yolo11_640_fire_11class": {"avg_ms": 46.113, "count": 140, "max_ms": 60.449, "p95_ms": 53.493}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.298, "count": 140, "max_ms": 82.315, "p95_ms": 73.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.503, "count": 141, "max_ms": 226.065, "p95_ms": 217.296}, "read_interval": {"avg_ms": 213.014, "count": 141, "max_ms": 235.495, "p95_ms": 226.505}, "triton_wait": {"avg_ms": 60.423, "count": 140, "max_ms": 82.327, "p95_ms": 73.828}, "visualize": {"avg_ms": 9.72, "count": 140, "max_ms": 12.628, "p95_ms": 10.637}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 60}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5894.34, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:25:33+09:00", "last_error_summary": "[h264 @ 0x558848702c80] error while decoding MB 41 39, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:25:33+09:00", "last_error_summary": "[h264 @ 0x558848702c80] error while decoding MB 41 39, bytestream -5", "last_exit_at": "2026-07-01T12:25:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.59, "last_started_at": "2026-07-01T12:23:25+09:00", "pid": 4904, "process": {"exists": false, "pid": 4904}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.946, "sample_count": 1366, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7291, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:25:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:25:33+09:00", "last_frame_read_age_sec": 0.215, "last_frame_read_at": "2026-07-01T12:25:33+09:00", "mjpeg": {"age_sec": 0.211, "exists": true, "mtime": "2026-07-01T12:25:33.088432+09:00", "mtime_epoch": 1782876333.0884323, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178231}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:25:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2353.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.725, "pushed": 4.725, "read": 4.692}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.27, "count": 141, "max_ms": 14.56, "p95_ms": 10.226}, "gstreamer_push": {"avg_ms": 0.845, "count": 141, "max_ms": 0.948, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.389, "count": 141, "max_ms": 43.845, "p95_ms": 38.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.539, "count": 141, "max_ms": 78.311, "p95_ms": 68.141}, "yolo11_640_fire_11class": {"avg_ms": 46.098, "count": 141, "max_ms": 60.449, "p95_ms": 53.493}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.279, "count": 141, "max_ms": 82.315, "p95_ms": 73.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.536, "count": 141, "max_ms": 226.065, "p95_ms": 217.296}, "read_interval": {"avg_ms": 212.99, "count": 140, "max_ms": 235.495, "p95_ms": 226.505}, "triton_wait": {"avg_ms": 60.404, "count": 141, "max_ms": 82.327, "p95_ms": 73.828}, "visualize": {"avg_ms": 9.733, "count": 141, "max_ms": 12.628, "p95_ms": 10.706}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.84, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 60}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5894.332, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:25:33+09:00", "last_error_summary": "[h264 @ 0x558848702c80] error while decoding MB 41 39, bytestream -5", "last_exit_at": "2026-07-01T12:25:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.594, "last_started_at": "2026-07-01T12:23:25+09:00", "pid": 4904, "process": {"exists": false, "pid": 4904}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.946, "sample_count": 1366, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7291, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:25:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T12:25:33+09:00", "last_frame_read_age_sec": 0.218, "last_frame_read_at": "2026-07-01T12:25:33+09:00", "mjpeg": {"age_sec": 0.215, "exists": true, "mtime": "2026-07-01T12:25:33.088432+09:00", "mtime_epoch": 1782876333.0884323, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178231}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:25:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2353.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.725, "pushed": 4.725, "read": 4.691}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.27, "count": 141, "max_ms": 14.56, "p95_ms": 10.226}, "gstreamer_push": {"avg_ms": 0.845, "count": 141, "max_ms": 0.948, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.389, "count": 141, "max_ms": 43.845, "p95_ms": 38.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.539, "count": 141, "max_ms": 78.311, "p95_ms": 68.141}, "yolo11_640_fire_11class": {"avg_ms": 46.098, "count": 141, "max_ms": 60.449, "p95_ms": 53.493}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.279, "count": 141, "max_ms": 82.315, "p95_ms": 73.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.536, "count": 141, "max_ms": 226.065, "p95_ms": 217.296}, "read_interval": {"avg_ms": 212.99, "count": 140, "max_ms": 235.495, "p95_ms": 226.505}, "triton_wait": {"avg_ms": 60.404, "count": 141, "max_ms": 82.327, "p95_ms": 73.828}, "visualize": {"avg_ms": 9.733, "count": 141, "max_ms": 12.628, "p95_ms": 10.706}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.844, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 60}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6002.434, "limit_mb": 8796093022207.996}, "detail": {"pid": 5011, "restart_count": 17, "start_count": 18}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:25:33+09:00", "last_error_summary": "[h264 @ 0x558848702c80] error while decoding MB 41 39, bytestream -5", "last_exit_at": "2026-07-01T12:25:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:25:35+09:00", "pid": 5011, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.406, "name": "ffmpeg", "pid": 5011, "rss_mb": 139.406, "state": "S (sleeping)", "threads": 106}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.946, "sample_count": 1346, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7291, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:25:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.428, "last_frame_pushed_at": "2026-07-01T12:25:33+09:00", "last_frame_read_age_sec": 4.64, "last_frame_read_at": "2026-07-01T12:25:33+09:00", "mjpeg": {"age_sec": 4.412, "exists": true, "mtime": "2026-07-01T12:25:33.312436+09:00", "mtime_epoch": 1782876333.312436, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178389}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:25:37+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2353.742, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.022, "pushed": 4.022, "read": 3.989}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.197, "count": 120, "max_ms": 11.068, "p95_ms": 10.133}, "gstreamer_push": {"avg_ms": 0.844, "count": 120, "max_ms": 0.948, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.498, "count": 120, "max_ms": 43.845, "p95_ms": 39.383}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.838, "count": 120, "max_ms": 78.311, "p95_ms": 70.451}, "yolo11_640_fire_11class": {"avg_ms": 46.066, "count": 120, "max_ms": 60.449, "p95_ms": 53.493}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.527, "count": 120, "max_ms": 82.315, "p95_ms": 75.101}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.985, "count": 120, "max_ms": 226.065, "p95_ms": 217.667}, "read_interval": {"avg_ms": 213.419, "count": 119, "max_ms": 235.495, "p95_ms": 231.008}, "triton_wait": {"avg_ms": 60.671, "count": 120, "max_ms": 82.327, "p95_ms": 75.114}, "visualize": {"avg_ms": 9.779, "count": 120, "max_ms": 12.628, "p95_ms": 10.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.833, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 65}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5895.863, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555ce417ebc0] error while decoding MB 74 16, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:27:43+09:00", "last_error_summary": "[h264 @ 0x555ce417ebc0] error while decoding MB 74 16, bytestream -7", "last_exit_at": "2026-07-01T12:25:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.456, "last_started_at": "2026-07-01T12:25:35+09:00", "pid": 5011, "process": {"exists": false, "pid": 5011}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.905, "sample_count": 1372, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7883, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:27:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.055, "last_frame_pushed_at": "2026-07-01T12:27:43+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T12:27:43+09:00", "mjpeg": {"age_sec": 0.033, "exists": true, "mtime": "2026-07-01T12:27:43.146577+09:00", "mtime_epoch": 1782876463.146577, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177850}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:27:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2354.0, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 143, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.777, "pushed": 4.777, "read": 4.777}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.063, "count": 142, "max_ms": 11.133, "p95_ms": 9.663}, "gstreamer_push": {"avg_ms": 0.854, "count": 143, "max_ms": 1.217, "p95_ms": 0.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.62, "count": 142, "max_ms": 46.234, "p95_ms": 42.169}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.383, "count": 142, "max_ms": 75.747, "p95_ms": 70.062}, "yolo11_640_fire_11class": {"avg_ms": 46.207, "count": 142, "max_ms": 61.692, "p95_ms": 57.238}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.342, "count": 142, "max_ms": 81.111, "p95_ms": 73.288}}, "truncated_count": 0}, "processing_total": {"avg_ms": 200.725, "count": 143, "max_ms": 225.04, "p95_ms": 218.56}, "read_interval": {"avg_ms": 210.52, "count": 143, "max_ms": 234.414, "p95_ms": 228.919}, "triton_wait": {"avg_ms": 60.396, "count": 142, "max_ms": 81.123, "p95_ms": 73.3}, "visualize": {"avg_ms": 9.526, "count": 142, "max_ms": 12.745, "p95_ms": 10.162}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 65}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5895.477, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:27:43+09:00", "last_error_summary": "[h264 @ 0x555ce417ebc0] error while decoding MB 74 16, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:27:43+09:00", "last_error_summary": "[h264 @ 0x555ce417ebc0] error while decoding MB 74 16, bytestream -7", "last_exit_at": "2026-07-01T12:27:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.619, "last_started_at": "2026-07-01T12:25:35+09:00", "pid": 5011, "process": {"exists": false, "pid": 5011}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.905, "sample_count": 1373, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7884, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:27:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:27:43+09:00", "last_frame_read_age_sec": 0.212, "last_frame_read_at": "2026-07-01T12:27:43+09:00", "mjpeg": {"age_sec": 0.196, "exists": true, "mtime": "2026-07-01T12:27:43.146577+09:00", "mtime_epoch": 1782876463.146577, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177850}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:27:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2354.0, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.785, "pushed": 4.785, "read": 4.751}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.078, "count": 142, "max_ms": 11.142, "p95_ms": 9.663}, "gstreamer_push": {"avg_ms": 0.854, "count": 143, "max_ms": 1.217, "p95_ms": 0.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.59, "count": 142, "max_ms": 46.234, "p95_ms": 42.169}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.32, "count": 142, "max_ms": 75.747, "p95_ms": 70.062}, "yolo11_640_fire_11class": {"avg_ms": 46.124, "count": 142, "max_ms": 61.692, "p95_ms": 57.238}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.281, "count": 142, "max_ms": 81.111, "p95_ms": 73.288}}, "truncated_count": 0}, "processing_total": {"avg_ms": 200.669, "count": 143, "max_ms": 225.04, "p95_ms": 218.56}, "read_interval": {"avg_ms": 210.42, "count": 142, "max_ms": 234.414, "p95_ms": 228.919}, "triton_wait": {"avg_ms": 60.334, "count": 142, "max_ms": 81.123, "p95_ms": 73.3}, "visualize": {"avg_ms": 9.54, "count": 142, "max_ms": 12.745, "p95_ms": 10.263}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 65}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5895.594, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:27:43+09:00", "last_error_summary": "[h264 @ 0x555ce417ebc0] error while decoding MB 74 16, bytestream -7", "last_exit_at": "2026-07-01T12:27:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.622, "last_started_at": "2026-07-01T12:25:35+09:00", "pid": 5011, "process": {"exists": false, "pid": 5011}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.905, "sample_count": 1373, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7884, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:27:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T12:27:43+09:00", "last_frame_read_age_sec": 0.215, "last_frame_read_at": "2026-07-01T12:27:43+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T12:27:43.146577+09:00", "mtime_epoch": 1782876463.146577, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177850}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:27:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2354.0, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.784, "pushed": 4.784, "read": 4.75}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.078, "count": 142, "max_ms": 11.142, "p95_ms": 9.663}, "gstreamer_push": {"avg_ms": 0.854, "count": 143, "max_ms": 1.217, "p95_ms": 0.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.59, "count": 142, "max_ms": 46.234, "p95_ms": 42.169}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.32, "count": 142, "max_ms": 75.747, "p95_ms": 70.062}, "yolo11_640_fire_11class": {"avg_ms": 46.124, "count": 142, "max_ms": 61.692, "p95_ms": 57.238}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.281, "count": 142, "max_ms": 81.111, "p95_ms": 73.288}}, "truncated_count": 0}, "processing_total": {"avg_ms": 200.669, "count": 143, "max_ms": 225.04, "p95_ms": 218.56}, "read_interval": {"avg_ms": 210.42, "count": 142, "max_ms": 234.414, "p95_ms": 228.919}, "triton_wait": {"avg_ms": 60.334, "count": 142, "max_ms": 81.123, "p95_ms": 73.3}, "visualize": {"avg_ms": 9.54, "count": 142, "max_ms": 12.745, "p95_ms": 10.263}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 65}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6000.879, "limit_mb": 8796093022207.996}, "detail": {"pid": 5118, "restart_count": 18, "start_count": 19}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:27:43+09:00", "last_error_summary": "[h264 @ 0x555ce417ebc0] error while decoding MB 74 16, bytestream -7", "last_exit_at": "2026-07-01T12:27:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:27:45+09:00", "pid": 5118, "process": {"exists": true, "fd_count": 5, "hwm_mb": 135.973, "name": "ffmpeg", "pid": 5118, "rss_mb": 135.973, "state": "S (sleeping)", "threads": 106}, "restart_count": 18, "restart_count_per_hour": 18, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.982, "sample_count": 1353, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7884, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:27:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.112, "last_frame_pushed_at": "2026-07-01T12:27:43+09:00", "last_frame_read_age_sec": 4.321, "last_frame_read_at": "2026-07-01T12:27:43+09:00", "mjpeg": {"age_sec": 4.097, "exists": true, "mtime": "2026-07-01T12:27:43.354580+09:00", "mtime_epoch": 1782876463.3545804, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177791}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:27:47+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2354.0, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.123, "pushed": 4.123, "read": 4.09}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.094, "count": 123, "max_ms": 11.142, "p95_ms": 9.663}, "gstreamer_push": {"avg_ms": 0.854, "count": 123, "max_ms": 1.217, "p95_ms": 0.892}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.402, "count": 123, "max_ms": 46.234, "p95_ms": 42.185}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.363, "count": 123, "max_ms": 75.747, "p95_ms": 70.062}, "yolo11_640_fire_11class": {"avg_ms": 46.47, "count": 123, "max_ms": 61.692, "p95_ms": 57.318}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.326, "count": 123, "max_ms": 81.111, "p95_ms": 73.667}}, "truncated_count": 0}, "processing_total": {"avg_ms": 200.847, "count": 123, "max_ms": 225.04, "p95_ms": 218.56}, "read_interval": {"avg_ms": 210.573, "count": 122, "max_ms": 234.414, "p95_ms": 228.919}, "triton_wait": {"avg_ms": 60.386, "count": 123, "max_ms": 81.123, "p95_ms": 75.035}, "visualize": {"avg_ms": 9.578, "count": 123, "max_ms": 12.745, "p95_ms": 10.263}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.831, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 70}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5896.832, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:27:43+09:00", "last_error_summary": "[h264 @ 0x555ce417ebc0] error while decoding MB 74 16, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:27:43+09:00", "last_error_summary": "[h264 @ 0x555ce417ebc0] error while decoding MB 74 16, bytestream -7", "last_exit_at": "2026-07-01T12:29:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.549, "last_started_at": "2026-07-01T12:27:45+09:00", "pid": 5118, "process": {"exists": false, "pid": 5118}, "restart_count": 18, "restart_count_per_hour": 18, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.592, "sample_count": 1368, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:29:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T12:29:52+09:00", "last_frame_read_age_sec": 0.205, "last_frame_read_at": "2026-07-01T12:29:52+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T12:29:52.800718+09:00", "mtime_epoch": 1782876592.8007183, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178850}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:29:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2354.516, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.723, "pushed": 4.723, "read": 4.69}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.292, "count": 141, "max_ms": 14.591, "p95_ms": 10.247}, "gstreamer_push": {"avg_ms": 0.849, "count": 141, "max_ms": 0.954, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.32, "count": 141, "max_ms": 49.558, "p95_ms": 42.096}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.827, "count": 141, "max_ms": 79.044, "p95_ms": 69.231}, "yolo11_640_fire_11class": {"avg_ms": 46.443, "count": 141, "max_ms": 65.114, "p95_ms": 58.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.8, "count": 141, "max_ms": 80.195, "p95_ms": 73.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.449, "count": 141, "max_ms": 224.321, "p95_ms": 217.777}, "read_interval": {"avg_ms": 213.228, "count": 140, "max_ms": 233.969, "p95_ms": 227.187}, "triton_wait": {"avg_ms": 60.229, "count": 141, "max_ms": 80.207, "p95_ms": 73.344}, "visualize": {"avg_ms": 9.755, "count": 141, "max_ms": 12.652, "p95_ms": 10.287}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.853, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 70}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5896.754, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:27:43+09:00", "last_error_summary": "[h264 @ 0x555ce417ebc0] error while decoding MB 74 16, bytestream -7", "last_exit_at": "2026-07-01T12:29:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.553, "last_started_at": "2026-07-01T12:27:45+09:00", "pid": 5118, "process": {"exists": false, "pid": 5118}, "restart_count": 18, "restart_count_per_hour": 18, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.592, "sample_count": 1368, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:29:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T12:29:52+09:00", "last_frame_read_age_sec": 0.209, "last_frame_read_at": "2026-07-01T12:29:52+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-01T12:29:52.800718+09:00", "mtime_epoch": 1782876592.8007183, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178850}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:29:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2354.516, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.723, "pushed": 4.723, "read": 4.689}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.292, "count": 141, "max_ms": 14.591, "p95_ms": 10.247}, "gstreamer_push": {"avg_ms": 0.849, "count": 141, "max_ms": 0.954, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.32, "count": 141, "max_ms": 49.558, "p95_ms": 42.096}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.827, "count": 141, "max_ms": 79.044, "p95_ms": 69.231}, "yolo11_640_fire_11class": {"avg_ms": 46.443, "count": 141, "max_ms": 65.114, "p95_ms": 58.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.8, "count": 141, "max_ms": 80.195, "p95_ms": 73.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.449, "count": 141, "max_ms": 224.321, "p95_ms": 217.777}, "read_interval": {"avg_ms": 213.228, "count": 140, "max_ms": 233.969, "p95_ms": 227.187}, "triton_wait": {"avg_ms": 60.229, "count": 141, "max_ms": 80.207, "p95_ms": 73.344}, "visualize": {"avg_ms": 9.755, "count": 141, "max_ms": 12.652, "p95_ms": 10.287}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.857, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 70}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6004.32, "limit_mb": 8796093022207.996}, "detail": {"pid": 5225, "restart_count": 19, "start_count": 20}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:27:43+09:00", "last_error_summary": "[h264 @ 0x555ce417ebc0] error while decoding MB 74 16, bytestream -7", "last_exit_at": "2026-07-01T12:29:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:29:55+09:00", "pid": 5225, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.105, "name": "ffmpeg", "pid": 5225, "rss_mb": 139.105, "state": "S (sleeping)", "threads": 106}, "restart_count": 19, "restart_count_per_hour": 19, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.592, "sample_count": 1347, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8470, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:29:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.662, "last_frame_pushed_at": "2026-07-01T12:29:52+09:00", "last_frame_read_age_sec": 4.863, "last_frame_read_at": "2026-07-01T12:29:52+09:00", "mjpeg": {"age_sec": 4.645, "exists": true, "mtime": "2026-07-01T12:29:53.012722+09:00", "mtime_epoch": 1782876593.0127218, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178927}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:29:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2354.516, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.992, "pushed": 3.992, "read": 3.958}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.28, "count": 119, "max_ms": 14.591, "p95_ms": 10.312}, "gstreamer_push": {"avg_ms": 0.848, "count": 119, "max_ms": 0.954, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.234, "count": 119, "max_ms": 49.558, "p95_ms": 40.742}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.951, "count": 119, "max_ms": 79.044, "p95_ms": 69.813}, "yolo11_640_fire_11class": {"avg_ms": 46.597, "count": 119, "max_ms": 65.114, "p95_ms": 59.783}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.875, "count": 119, "max_ms": 80.195, "p95_ms": 73.337}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.313, "count": 119, "max_ms": 224.321, "p95_ms": 217.939}, "read_interval": {"avg_ms": 213.073, "count": 118, "max_ms": 233.969, "p95_ms": 227.325}, "triton_wait": {"avg_ms": 60.276, "count": 119, "max_ms": 80.207, "p95_ms": 74.654}, "visualize": {"avg_ms": 9.703, "count": 119, "max_ms": 12.574, "p95_ms": 10.11}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.809, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 75}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5898.035, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5627f69d78c0] error while decoding MB 11 25, bytestream -17", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:32:00+09:00", "last_error_summary": "[h264 @ 0x5627f69d78c0] error while decoding MB 11 25, bytestream -17", "last_exit_at": "2026-07-01T12:29:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.638, "last_started_at": "2026-07-01T12:29:55+09:00", "pid": 5225, "process": {"exists": false, "pid": 5225}, "restart_count": 19, "restart_count_per_hour": 19, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.496, "sample_count": 1362, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9042, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:32:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T12:32:00+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T12:32:00+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T12:32:00.250826+09:00", "mtime_epoch": 1782876720.2508261, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179703}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:32:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2354.773, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.668, "pushed": 4.668, "read": 4.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.215, "count": 140, "max_ms": 10.884, "p95_ms": 10.514}, "gstreamer_push": {"avg_ms": 0.849, "count": 140, "max_ms": 0.915, "p95_ms": 0.881}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.615, "count": 140, "max_ms": 46.078, "p95_ms": 38.262}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.727, "count": 140, "max_ms": 75.079, "p95_ms": 71.131}, "yolo11_640_fire_11class": {"avg_ms": 48.025, "count": 140, "max_ms": 62.038, "p95_ms": 55.832}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.978, "count": 140, "max_ms": 83.539, "p95_ms": 75.83}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.23, "count": 140, "max_ms": 226.186, "p95_ms": 216.547}, "read_interval": {"avg_ms": 213.902, "count": 141, "max_ms": 235.829, "p95_ms": 226.279}, "triton_wait": {"avg_ms": 62.392, "count": 140, "max_ms": 83.552, "p95_ms": 75.841}, "visualize": {"avg_ms": 9.675, "count": 140, "max_ms": 11.853, "p95_ms": 10.705}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 75}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5896.848, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:32:00+09:00", "last_error_summary": "[h264 @ 0x5627f69d78c0] error while decoding MB 11 25, bytestream -17", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:32:00+09:00", "last_error_summary": "[h264 @ 0x5627f69d78c0] error while decoding MB 11 25, bytestream -17", "last_exit_at": "2026-07-01T12:32:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.805, "last_started_at": "2026-07-01T12:29:55+09:00", "pid": 5225, "process": {"exists": false, "pid": 5225}, "restart_count": 19, "restart_count_per_hour": 19, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.496, "sample_count": 1363, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9043, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:32:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:32:00+09:00", "last_frame_read_age_sec": 0.217, "last_frame_read_at": "2026-07-01T12:32:00+09:00", "mjpeg": {"age_sec": 0.21, "exists": true, "mtime": "2026-07-01T12:32:00.250826+09:00", "mtime_epoch": 1782876720.2508261, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179703}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:32:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2354.773, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.708, "pushed": 4.708, "read": 4.675}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.214, "count": 140, "max_ms": 10.884, "p95_ms": 10.514}, "gstreamer_push": {"avg_ms": 0.849, "count": 141, "max_ms": 0.915, "p95_ms": 0.881}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.611, "count": 140, "max_ms": 46.078, "p95_ms": 38.262}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.655, "count": 140, "max_ms": 75.079, "p95_ms": 70.76}, "yolo11_640_fire_11class": {"avg_ms": 48.067, "count": 140, "max_ms": 62.038, "p95_ms": 55.832}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.925, "count": 140, "max_ms": 83.539, "p95_ms": 75.83}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.291, "count": 141, "max_ms": 226.186, "p95_ms": 216.547}, "read_interval": {"avg_ms": 213.884, "count": 140, "max_ms": 235.829, "p95_ms": 226.279}, "triton_wait": {"avg_ms": 62.333, "count": 140, "max_ms": 83.552, "p95_ms": 75.841}, "visualize": {"avg_ms": 9.677, "count": 140, "max_ms": 11.853, "p95_ms": 10.705}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 75}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5896.965, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:32:00+09:00", "last_error_summary": "[h264 @ 0x5627f69d78c0] error while decoding MB 11 25, bytestream -17", "last_exit_at": "2026-07-01T12:32:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.808, "last_started_at": "2026-07-01T12:29:55+09:00", "pid": 5225, "process": {"exists": false, "pid": 5225}, "restart_count": 19, "restart_count_per_hour": 19, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.496, "sample_count": 1363, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9043, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:32:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T12:32:00+09:00", "last_frame_read_age_sec": 0.221, "last_frame_read_at": "2026-07-01T12:32:00+09:00", "mjpeg": {"age_sec": 0.214, "exists": true, "mtime": "2026-07-01T12:32:00.250826+09:00", "mtime_epoch": 1782876720.2508261, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179703}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:32:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2354.773, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.708, "pushed": 4.708, "read": 4.675}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.214, "count": 140, "max_ms": 10.884, "p95_ms": 10.514}, "gstreamer_push": {"avg_ms": 0.849, "count": 141, "max_ms": 0.915, "p95_ms": 0.881}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.611, "count": 140, "max_ms": 46.078, "p95_ms": 38.262}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.655, "count": 140, "max_ms": 75.079, "p95_ms": 70.76}, "yolo11_640_fire_11class": {"avg_ms": 48.067, "count": 140, "max_ms": 62.038, "p95_ms": 55.832}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.925, "count": 140, "max_ms": 83.539, "p95_ms": 75.83}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.291, "count": 141, "max_ms": 226.186, "p95_ms": 216.547}, "read_interval": {"avg_ms": 213.884, "count": 140, "max_ms": 235.829, "p95_ms": 226.279}, "triton_wait": {"avg_ms": 62.333, "count": 140, "max_ms": 83.552, "p95_ms": 75.841}, "visualize": {"avg_ms": 9.677, "count": 140, "max_ms": 11.853, "p95_ms": 10.705}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 75}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6005.402, "limit_mb": 8796093022207.996}, "detail": {"pid": 5332, "restart_count": 20, "start_count": 21}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:32:00+09:00", "last_error_summary": "[h264 @ 0x5627f69d78c0] error while decoding MB 11 25, bytestream -17", "last_exit_at": "2026-07-01T12:32:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:32:03+09:00", "pid": 5332, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.008, "name": "ffmpeg", "pid": 5332, "rss_mb": 139.008, "state": "S (sleeping)", "threads": 106}, "restart_count": 20, "restart_count_per_hour": 20, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.592, "sample_count": 1340, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9043, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:32:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.7, "last_frame_pushed_at": "2026-07-01T12:32:00+09:00", "last_frame_read_age_sec": 4.914, "last_frame_read_at": "2026-07-01T12:32:00+09:00", "mjpeg": {"age_sec": 4.684, "exists": true, "mtime": "2026-07-01T12:32:00.474830+09:00", "mtime_epoch": 1782876720.47483, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179998}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:32:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2354.773, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.98, "pushed": 3.98, "read": 3.947}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.193, "count": 118, "max_ms": 10.884, "p95_ms": 10.618}, "gstreamer_push": {"avg_ms": 0.848, "count": 119, "max_ms": 0.915, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.456, "count": 118, "max_ms": 46.078, "p95_ms": 38.911}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.659, "count": 118, "max_ms": 75.079, "p95_ms": 71.131}, "yolo11_640_fire_11class": {"avg_ms": 47.879, "count": 118, "max_ms": 61.257, "p95_ms": 56.73}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.021, "count": 118, "max_ms": 80.151, "p95_ms": 76.3}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.866, "count": 119, "max_ms": 224.613, "p95_ms": 217.081}, "read_interval": {"avg_ms": 213.446, "count": 118, "max_ms": 235.352, "p95_ms": 226.665}, "triton_wait": {"avg_ms": 62.409, "count": 118, "max_ms": 80.163, "p95_ms": 76.311}, "visualize": {"avg_ms": 9.685, "count": 118, "max_ms": 11.853, "p95_ms": 10.752}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 80}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5899.121, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x565259084c40] error while decoding MB 5 26, bytestream -36", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:34:09+09:00", "last_error_summary": "[h264 @ 0x565259084c40] error while decoding MB 5 26, bytestream -36", "last_exit_at": "2026-07-01T12:32:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.86, "last_started_at": "2026-07-01T12:32:03+09:00", "pid": 5332, "process": {"exists": false, "pid": 5332}, "restart_count": 20, "restart_count_per_hour": 20, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.321, "sample_count": 1348, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9612, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:34:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T12:34:08+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T12:34:08+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T12:34:08.976957+09:00", "mtime_epoch": 1782876848.976957, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182343}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:34:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.031, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.664, "pushed": 4.664, "read": 4.664}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.334, "count": 139, "max_ms": 11.624, "p95_ms": 10.482}, "gstreamer_push": {"avg_ms": 0.846, "count": 139, "max_ms": 1.316, "p95_ms": 0.885}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.169, "count": 139, "max_ms": 44.465, "p95_ms": 39.706}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.637, "count": 139, "max_ms": 78.364, "p95_ms": 68.899}, "yolo11_640_fire_11class": {"avg_ms": 46.03, "count": 139, "max_ms": 59.913, "p95_ms": 52.856}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.384, "count": 139, "max_ms": 78.827, "p95_ms": 73.61}}, "truncated_count": 0}, "processing_total": {"avg_ms": 205.803, "count": 139, "max_ms": 228.724, "p95_ms": 222.62}, "read_interval": {"avg_ms": 215.511, "count": 139, "max_ms": 239.475, "p95_ms": 231.932}, "triton_wait": {"avg_ms": 61.181, "count": 139, "max_ms": 78.84, "p95_ms": 74.519}, "visualize": {"avg_ms": 9.743, "count": 139, "max_ms": 13.55, "p95_ms": 10.802}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.804, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 80}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5898.477, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:34:09+09:00", "last_error_summary": "[h264 @ 0x565259084c40] error while decoding MB 5 26, bytestream -36", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:34:09+09:00", "last_error_summary": "[h264 @ 0x565259084c40] error while decoding MB 5 26, bytestream -36", "last_exit_at": "2026-07-01T12:34:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.041, "last_started_at": "2026-07-01T12:32:03+09:00", "pid": 5332, "process": {"exists": false, "pid": 5332}, "restart_count": 20, "restart_count_per_hour": 20, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.488, "sample_count": 1348, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9613, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:34:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:34:09+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T12:34:08+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T12:34:08.976957+09:00", "mtime_epoch": 1782876848.976957, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182343}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:34:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.031, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.669, "pushed": 4.669, "read": 4.636}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.345, "count": 139, "max_ms": 11.624, "p95_ms": 10.494}, "gstreamer_push": {"avg_ms": 0.846, "count": 140, "max_ms": 1.316, "p95_ms": 0.885}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.15, "count": 139, "max_ms": 44.465, "p95_ms": 39.706}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.579, "count": 139, "max_ms": 78.364, "p95_ms": 68.899}, "yolo11_640_fire_11class": {"avg_ms": 45.978, "count": 139, "max_ms": 59.913, "p95_ms": 52.856}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.34, "count": 139, "max_ms": 78.827, "p95_ms": 73.61}}, "truncated_count": 0}, "processing_total": {"avg_ms": 205.948, "count": 140, "max_ms": 228.724, "p95_ms": 222.62}, "read_interval": {"avg_ms": 215.511, "count": 139, "max_ms": 239.475, "p95_ms": 231.932}, "triton_wait": {"avg_ms": 61.137, "count": 139, "max_ms": 78.84, "p95_ms": 74.519}, "visualize": {"avg_ms": 9.768, "count": 139, "max_ms": 13.55, "p95_ms": 10.814}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 80}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5898.594, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:34:09+09:00", "last_error_summary": "[h264 @ 0x565259084c40] error while decoding MB 5 26, bytestream -36", "last_exit_at": "2026-07-01T12:34:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.045, "last_started_at": "2026-07-01T12:32:03+09:00", "pid": 5332, "process": {"exists": false, "pid": 5332}, "restart_count": 20, "restart_count_per_hour": 20, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.488, "sample_count": 1348, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9613, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:34:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T12:34:09+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T12:34:08+09:00", "mjpeg": {"age_sec": 0.225, "exists": true, "mtime": "2026-07-01T12:34:08.976957+09:00", "mtime_epoch": 1782876848.976957, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182343}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:34:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.031, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.669, "pushed": 4.669, "read": 4.635}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.345, "count": 139, "max_ms": 11.624, "p95_ms": 10.494}, "gstreamer_push": {"avg_ms": 0.846, "count": 140, "max_ms": 1.316, "p95_ms": 0.885}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.15, "count": 139, "max_ms": 44.465, "p95_ms": 39.706}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.579, "count": 139, "max_ms": 78.364, "p95_ms": 68.899}, "yolo11_640_fire_11class": {"avg_ms": 45.978, "count": 139, "max_ms": 59.913, "p95_ms": 52.856}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.34, "count": 139, "max_ms": 78.827, "p95_ms": 73.61}}, "truncated_count": 0}, "processing_total": {"avg_ms": 205.948, "count": 140, "max_ms": 228.724, "p95_ms": 222.62}, "read_interval": {"avg_ms": 215.511, "count": 139, "max_ms": 239.475, "p95_ms": 231.932}, "triton_wait": {"avg_ms": 61.137, "count": 139, "max_ms": 78.84, "p95_ms": 74.519}, "visualize": {"avg_ms": 9.768, "count": 139, "max_ms": 13.55, "p95_ms": 10.814}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 80}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6006.488, "limit_mb": 8796093022207.996}, "detail": {"pid": 5439, "restart_count": 21, "start_count": 22}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:34:09+09:00", "last_error_summary": "[h264 @ 0x565259084c40] error while decoding MB 5 26, bytestream -36", "last_exit_at": "2026-07-01T12:34:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:34:12+09:00", "pid": 5439, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.477, "name": "ffmpeg", "pid": 5439, "rss_mb": 139.477, "state": "S (sleeping)", "threads": 106}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.488, "sample_count": 1325, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9613, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:34:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.901, "last_frame_pushed_at": "2026-07-01T12:34:09+09:00", "last_frame_read_age_sec": 5.128, "last_frame_read_at": "2026-07-01T12:34:08+09:00", "mjpeg": {"age_sec": 4.879, "exists": true, "mtime": "2026-07-01T12:34:09.216961+09:00", "mtime_epoch": 1782876849.2169611, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182869}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:34:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.031, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.905, "pushed": 3.905, "read": 3.872}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.366, "count": 116, "max_ms": 11.624, "p95_ms": 10.503}, "gstreamer_push": {"avg_ms": 0.848, "count": 117, "max_ms": 1.316, "p95_ms": 0.892}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.119, "count": 116, "max_ms": 43.362, "p95_ms": 39.706}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.528, "count": 116, "max_ms": 78.364, "p95_ms": 66.472}, "yolo11_640_fire_11class": {"avg_ms": 46.108, "count": 116, "max_ms": 59.041, "p95_ms": 52.664}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.312, "count": 116, "max_ms": 78.827, "p95_ms": 73.61}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.255, "count": 117, "max_ms": 228.724, "p95_ms": 222.849}, "read_interval": {"avg_ms": 215.775, "count": 116, "max_ms": 239.475, "p95_ms": 231.935}, "triton_wait": {"avg_ms": 61.089, "count": 116, "max_ms": 78.84, "p95_ms": 74.29}, "visualize": {"avg_ms": 9.782, "count": 116, "max_ms": 13.55, "p95_ms": 10.802}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_123419_530b01ae.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 6190.633, "limit_mb": 8796093022207.996}, "detail": {"pid": 5549, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:34:19+09:00", "pid": 5549, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.84, "name": "ffmpeg", "pid": 5549, "rss_mb": 137.84, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.802}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "530b01ae", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3183.854, "exists": true, "mtime": "2026-07-01T11:41:17.732468+09:00", "mtime_epoch": 1782873677.7324684, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 263066}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T12:34:21+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 703.281, "name": "celery", "pid": 239, "rss_mb": 668.922, "state": "R (running)", "threads": 118}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T12:34:19+09:00", "state": "running", "task_id": "07da80a4-4fde-44d2-8f35-7ab9b2b2637e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 84}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6178.879, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559e66694380] error while decoding MB 69 32, bytestream -8", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:36:18+09:00", "last_error_summary": "[h264 @ 0x559e66694380] error while decoding MB 69 32, bytestream -8", "last_exit_at": "2026-07-01T12:34:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.237, "last_started_at": "2026-07-01T12:34:12+09:00", "pid": 5439, "process": {"exists": false, "pid": 5439}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.732, "sample_count": 1320, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10167, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:36:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T12:36:18+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T12:36:18+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T12:36:18.291100+09:00", "mtime_epoch": 1782876978.2910995, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182430}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:36:18+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.289, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.395, "pushed": 4.395, "read": 4.395}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.968, "count": 131, "max_ms": 15.486, "p95_ms": 13.276}, "gstreamer_push": {"avg_ms": 0.874, "count": 131, "max_ms": 1.384, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.607, "count": 131, "max_ms": 46.761, "p95_ms": 41.017}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.54, "count": 131, "max_ms": 98.273, "p95_ms": 89.479}, "yolo11_640_fire_11class": {"avg_ms": 48.37, "count": 131, "max_ms": 67.608, "p95_ms": 60.376}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.009, "count": 131, "max_ms": 91.545, "p95_ms": 79.333}}, "truncated_count": 0}, "processing_total": {"avg_ms": 218.894, "count": 131, "max_ms": 281.604, "p95_ms": 245.053}, "read_interval": {"avg_ms": 228.675, "count": 131, "max_ms": 292.177, "p95_ms": 254.636}, "triton_wait": {"avg_ms": 67.077, "count": 131, "max_ms": 98.288, "p95_ms": 89.492}, "visualize": {"avg_ms": 10.116, "count": 131, "max_ms": 14.672, "p95_ms": 12.592}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.805, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 84}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6166.895, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:36:18+09:00", "last_error_summary": "[h264 @ 0x559e66694380] error while decoding MB 69 32, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:36:18+09:00", "last_error_summary": "[h264 @ 0x559e66694380] error while decoding MB 69 32, bytestream -8", "last_exit_at": "2026-07-01T12:36:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.401, "last_started_at": "2026-07-01T12:34:12+09:00", "pid": 5439, "process": {"exists": false, "pid": 5439}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.732, "sample_count": 1320, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10168, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:36:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:36:18+09:00", "last_frame_read_age_sec": 0.216, "last_frame_read_at": "2026-07-01T12:36:18+09:00", "mjpeg": {"age_sec": 0.206, "exists": true, "mtime": "2026-07-01T12:36:18.291100+09:00", "mtime_epoch": 1782876978.2910995, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182430}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:36:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.289, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.405, "pushed": 4.405, "read": 4.371}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.988, "count": 131, "max_ms": 15.486, "p95_ms": 13.276}, "gstreamer_push": {"avg_ms": 0.874, "count": 132, "max_ms": 1.384, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.577, "count": 131, "max_ms": 46.761, "p95_ms": 41.017}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.542, "count": 131, "max_ms": 98.273, "p95_ms": 89.479}, "yolo11_640_fire_11class": {"avg_ms": 48.35, "count": 131, "max_ms": 67.608, "p95_ms": 60.376}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.997, "count": 131, "max_ms": 91.545, "p95_ms": 79.333}}, "truncated_count": 0}, "processing_total": {"avg_ms": 218.835, "count": 132, "max_ms": 281.604, "p95_ms": 245.053}, "read_interval": {"avg_ms": 228.675, "count": 131, "max_ms": 292.177, "p95_ms": 254.636}, "triton_wait": {"avg_ms": 67.066, "count": 131, "max_ms": 98.288, "p95_ms": 89.492}, "visualize": {"avg_ms": 10.132, "count": 131, "max_ms": 14.672, "p95_ms": 12.592}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 84}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6166.883, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:36:18+09:00", "last_error_summary": "[h264 @ 0x559e66694380] error while decoding MB 69 32, bytestream -8", "last_exit_at": "2026-07-01T12:36:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.405, "last_started_at": "2026-07-01T12:34:12+09:00", "pid": 5439, "process": {"exists": false, "pid": 5439}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.732, "sample_count": 1320, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10168, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:36:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T12:36:18+09:00", "last_frame_read_age_sec": 0.219, "last_frame_read_at": "2026-07-01T12:36:18+09:00", "mjpeg": {"age_sec": 0.209, "exists": true, "mtime": "2026-07-01T12:36:18.291100+09:00", "mtime_epoch": 1782876978.2910995, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182430}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:36:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.289, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.404, "pushed": 4.404, "read": 4.371}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.988, "count": 131, "max_ms": 15.486, "p95_ms": 13.276}, "gstreamer_push": {"avg_ms": 0.874, "count": 132, "max_ms": 1.384, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.577, "count": 131, "max_ms": 46.761, "p95_ms": 41.017}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.542, "count": 131, "max_ms": 98.273, "p95_ms": 89.479}, "yolo11_640_fire_11class": {"avg_ms": 48.35, "count": 131, "max_ms": 67.608, "p95_ms": 60.376}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.997, "count": 131, "max_ms": 91.545, "p95_ms": 79.333}}, "truncated_count": 0}, "processing_total": {"avg_ms": 218.835, "count": 132, "max_ms": 281.604, "p95_ms": 245.053}, "read_interval": {"avg_ms": 228.675, "count": 131, "max_ms": 292.177, "p95_ms": 254.636}, "triton_wait": {"avg_ms": 67.066, "count": 131, "max_ms": 98.288, "p95_ms": 89.492}, "visualize": {"avg_ms": 10.132, "count": 131, "max_ms": 14.672, "p95_ms": 12.592}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 84}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6280.055, "limit_mb": 8796093022207.996}, "detail": {"pid": 5702, "restart_count": 22, "start_count": 23}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:36:18+09:00", "last_error_summary": "[h264 @ 0x559e66694380] error while decoding MB 69 32, bytestream -8", "last_exit_at": "2026-07-01T12:36:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:36:21+09:00", "pid": 5702, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.621, "name": "ffmpeg", "pid": 5702, "rss_mb": 138.621, "state": "S (sleeping)", "threads": 106}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.77, "sample_count": 1298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10168, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:36:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.798, "last_frame_pushed_at": "2026-07-01T12:36:18+09:00", "last_frame_read_age_sec": 5.011, "last_frame_read_at": "2026-07-01T12:36:18+09:00", "mjpeg": {"age_sec": 4.781, "exists": true, "mtime": "2026-07-01T12:36:18.511103+09:00", "mtime_epoch": 1782876978.5111032, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180953}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:36:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.289, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.704, "pushed": 3.704, "read": 3.671}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.026, "count": 110, "max_ms": 15.486, "p95_ms": 13.275}, "gstreamer_push": {"avg_ms": 0.874, "count": 111, "max_ms": 1.384, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.628, "count": 110, "max_ms": 46.761, "p95_ms": 42.225}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.154, "count": 110, "max_ms": 98.273, "p95_ms": 88.619}, "yolo11_640_fire_11class": {"avg_ms": 48.502, "count": 110, "max_ms": 67.608, "p95_ms": 60.721}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.036, "count": 110, "max_ms": 91.545, "p95_ms": 78.086}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.099, "count": 111, "max_ms": 281.604, "p95_ms": 245.053}, "read_interval": {"avg_ms": 228.874, "count": 110, "max_ms": 292.177, "p95_ms": 254.636}, "triton_wait": {"avg_ms": 66.897, "count": 110, "max_ms": 98.288, "p95_ms": 88.78}, "visualize": {"avg_ms": 10.223, "count": 110, "max_ms": 14.672, "p95_ms": 12.592}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_123636_206b0021.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 6251.98, "limit_mb": 8796093022207.996}, "detail": {"pid": 5811, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:36:36+09:00", "pid": 5811, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.461, "name": "ffmpeg", "pid": 5811, "rss_mb": 137.461, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.79}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "206b0021", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.222, "exists": true, "mtime": "2026-07-01T12:36:35.331382+09:00", "mtime_epoch": 1782876995.331382, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 265669}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T12:36:38+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 703.281, "name": "celery", "pid": 239, "rss_mb": 668.895, "state": "R (running)", "threads": 118}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T12:36:36+09:00", "state": "running", "task_id": "c496d733-3612-43d0-9fe5-2118334c4f3b", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 88}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6173.988, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b7b22606c0] error while decoding MB 72 22, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:38:09+09:00", "last_error_summary": "[h264 @ 0x55b7b22606c0] error while decoding MB 72 22, bytestream -5", "last_exit_at": "2026-07-01T12:36:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.634, "last_started_at": "2026-07-01T12:36:21+09:00", "pid": 5702, "process": {"exists": false, "pid": 5702}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 243.764, "sample_count": 1297, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10637, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:38:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-01T12:38:09+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T12:38:09+09:00", "mjpeg": {"age_sec": 0.065, "exists": true, "mtime": "2026-07-01T12:38:09.860949+09:00", "mtime_epoch": 1782877089.860949, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183039}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:38:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.547, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.345, "pushed": 4.345, "read": 4.345}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.991, "count": 129, "max_ms": 14.744, "p95_ms": 12.407}, "gstreamer_push": {"avg_ms": 0.876, "count": 130, "max_ms": 1.372, "p95_ms": 0.966}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.483, "count": 129, "max_ms": 48.551, "p95_ms": 43.373}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.375, "count": 129, "max_ms": 104.207, "p95_ms": 86.917}, "yolo11_640_fire_11class": {"avg_ms": 48.069, "count": 129, "max_ms": 70.724, "p95_ms": 58.986}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.636, "count": 129, "max_ms": 91.768, "p95_ms": 78.948}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.477, "count": 130, "max_ms": 260.9, "p95_ms": 250.025}, "read_interval": {"avg_ms": 231.384, "count": 130, "max_ms": 271.221, "p95_ms": 260.986}, "triton_wait": {"avg_ms": 66.907, "count": 129, "max_ms": 104.222, "p95_ms": 88.049}, "visualize": {"avg_ms": 10.287, "count": 129, "max_ms": 13.812, "p95_ms": 11.522}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 88}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6176.25, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:38:09+09:00", "last_error_summary": "[h264 @ 0x55b7b22606c0] error while decoding MB 72 22, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:38:09+09:00", "last_error_summary": "[h264 @ 0x55b7b22606c0] error while decoding MB 72 22, bytestream -5", "last_exit_at": "2026-07-01T12:38:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.784, "last_started_at": "2026-07-01T12:36:21+09:00", "pid": 5702, "process": {"exists": false, "pid": 5702}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 243.764, "sample_count": 1297, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10638, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:38:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:38:10+09:00", "last_frame_read_age_sec": 0.22, "last_frame_read_at": "2026-07-01T12:38:09+09:00", "mjpeg": {"age_sec": 0.215, "exists": true, "mtime": "2026-07-01T12:38:09.860949+09:00", "mtime_epoch": 1782877089.860949, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183039}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:38:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.547, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.353, "pushed": 4.353, "read": 4.32}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.994, "count": 130, "max_ms": 14.744, "p95_ms": 12.407}, "gstreamer_push": {"avg_ms": 0.876, "count": 130, "max_ms": 1.372, "p95_ms": 0.966}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.462, "count": 130, "max_ms": 48.551, "p95_ms": 43.373}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.324, "count": 130, "max_ms": 104.207, "p95_ms": 86.917}, "yolo11_640_fire_11class": {"avg_ms": 48.034, "count": 130, "max_ms": 70.724, "p95_ms": 58.986}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.597, "count": 130, "max_ms": 91.768, "p95_ms": 78.948}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.378, "count": 130, "max_ms": 260.9, "p95_ms": 250.025}, "read_interval": {"avg_ms": 231.334, "count": 129, "max_ms": 271.221, "p95_ms": 260.986}, "triton_wait": {"avg_ms": 66.851, "count": 130, "max_ms": 104.222, "p95_ms": 88.049}, "visualize": {"avg_ms": 10.322, "count": 130, "max_ms": 14.835, "p95_ms": 11.663}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.862, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 88}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6176.242, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:38:09+09:00", "last_error_summary": "[h264 @ 0x55b7b22606c0] error while decoding MB 72 22, bytestream -5", "last_exit_at": "2026-07-01T12:38:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.788, "last_started_at": "2026-07-01T12:36:21+09:00", "pid": 5702, "process": {"exists": false, "pid": 5702}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 243.764, "sample_count": 1297, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10638, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:38:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:38:10+09:00", "last_frame_read_age_sec": 0.223, "last_frame_read_at": "2026-07-01T12:38:09+09:00", "mjpeg": {"age_sec": 0.218, "exists": true, "mtime": "2026-07-01T12:38:09.860949+09:00", "mtime_epoch": 1782877089.860949, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183039}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:38:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.547, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.353, "pushed": 4.353, "read": 4.319}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.994, "count": 130, "max_ms": 14.744, "p95_ms": 12.407}, "gstreamer_push": {"avg_ms": 0.876, "count": 130, "max_ms": 1.372, "p95_ms": 0.966}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.462, "count": 130, "max_ms": 48.551, "p95_ms": 43.373}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.324, "count": 130, "max_ms": 104.207, "p95_ms": 86.917}, "yolo11_640_fire_11class": {"avg_ms": 48.034, "count": 130, "max_ms": 70.724, "p95_ms": 58.986}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.597, "count": 130, "max_ms": 91.768, "p95_ms": 78.948}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.378, "count": 130, "max_ms": 260.9, "p95_ms": 250.025}, "read_interval": {"avg_ms": 231.334, "count": 129, "max_ms": 271.221, "p95_ms": 260.986}, "triton_wait": {"avg_ms": 66.851, "count": 130, "max_ms": 104.222, "p95_ms": 88.049}, "visualize": {"avg_ms": 10.322, "count": 130, "max_ms": 14.835, "p95_ms": 11.663}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 88}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6289.492, "limit_mb": 8796093022207.996}, "detail": {"pid": 5964, "restart_count": 23, "start_count": 24}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:38:09+09:00", "last_error_summary": "[h264 @ 0x55b7b22606c0] error while decoding MB 72 22, bytestream -5", "last_exit_at": "2026-07-01T12:38:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:38:12+09:00", "pid": 5964, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.039, "name": "ffmpeg", "pid": 5964, "rss_mb": 139.039, "state": "S (sleeping)", "threads": 106}, "restart_count": 23, "restart_count_per_hour": 23, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.241, "sample_count": 1276, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10638, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:38:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.557, "last_frame_pushed_at": "2026-07-01T12:38:10+09:00", "last_frame_read_age_sec": 4.774, "last_frame_read_at": "2026-07-01T12:38:09+09:00", "mjpeg": {"age_sec": 4.541, "exists": true, "mtime": "2026-07-01T12:38:10.088953+09:00", "mtime_epoch": 1782877090.0889528, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:38:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.547, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.671, "pushed": 3.671, "read": 3.638}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.985, "count": 109, "max_ms": 13.834, "p95_ms": 11.959}, "gstreamer_push": {"avg_ms": 0.878, "count": 110, "max_ms": 1.372, "p95_ms": 0.997}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.739, "count": 109, "max_ms": 48.551, "p95_ms": 44.276}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.347, "count": 109, "max_ms": 104.207, "p95_ms": 85.375}, "yolo11_640_fire_11class": {"avg_ms": 47.863, "count": 109, "max_ms": 70.724, "p95_ms": 58.986}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.777, "count": 109, "max_ms": 91.768, "p95_ms": 80.315}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.0, "count": 110, "max_ms": 260.9, "p95_ms": 250.025}, "read_interval": {"avg_ms": 232.977, "count": 109, "max_ms": 271.221, "p95_ms": 260.986}, "triton_wait": {"avg_ms": 67.065, "count": 109, "max_ms": 104.222, "p95_ms": 86.929}, "visualize": {"avg_ms": 10.394, "count": 109, "max_ms": 14.835, "p95_ms": 11.663}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 92}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6225.668, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5575c9f88cc0] error while decoding MB 41 19, bytestream -30", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:40:01+09:00", "last_error_summary": "[h264 @ 0x5575c9f88cc0] error while decoding MB 41 19, bytestream -30", "last_exit_at": "2026-07-01T12:38:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.047, "last_started_at": "2026-07-01T12:38:12+09:00", "pid": 5964, "process": {"exists": false, "pid": 5964}, "restart_count": 23, "restart_count_per_hour": 23, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.636, "sample_count": 1256, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11083, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:40:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T12:40:01+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T12:40:01+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T12:40:01.632244+09:00", "mtime_epoch": 1782877201.6322439, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188237}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:40:01+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.805, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.303, "pushed": 4.303, "read": 4.303}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.332, "count": 128, "max_ms": 16.207, "p95_ms": 13.721}, "gstreamer_push": {"avg_ms": 0.865, "count": 129, "max_ms": 1.181, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.713, "count": 128, "max_ms": 47.216, "p95_ms": 43.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.462, "count": 128, "max_ms": 98.92, "p95_ms": 86.784}, "yolo11_640_fire_11class": {"avg_ms": 48.931, "count": 128, "max_ms": 76.031, "p95_ms": 65.154}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.048, "count": 128, "max_ms": 94.586, "p95_ms": 81.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.951, "count": 129, "max_ms": 277.68, "p95_ms": 254.731}, "read_interval": {"avg_ms": 233.729, "count": 129, "max_ms": 286.92, "p95_ms": 264.17}, "triton_wait": {"avg_ms": 66.763, "count": 128, "max_ms": 98.933, "p95_ms": 87.885}, "visualize": {"avg_ms": 10.472, "count": 128, "max_ms": 14.439, "p95_ms": 12.666}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 92}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6224.84, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:40:01+09:00", "last_error_summary": "[h264 @ 0x5575c9f88cc0] error while decoding MB 41 19, bytestream -30", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:40:01+09:00", "last_error_summary": "[h264 @ 0x5575c9f88cc0] error while decoding MB 41 19, bytestream -30", "last_exit_at": "2026-07-01T12:40:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.228, "last_started_at": "2026-07-01T12:38:12+09:00", "pid": 5964, "process": {"exists": false, "pid": 5964}, "restart_count": 23, "restart_count_per_hour": 23, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.636, "sample_count": 1256, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11084, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:40:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:40:01+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T12:40:01+09:00", "mjpeg": {"age_sec": 0.225, "exists": true, "mtime": "2026-07-01T12:40:01.632244+09:00", "mtime_epoch": 1782877201.6322439, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188237}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:40:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.805, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.311, "pushed": 4.311, "read": 4.278}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.342, "count": 128, "max_ms": 16.207, "p95_ms": 13.721}, "gstreamer_push": {"avg_ms": 0.865, "count": 129, "max_ms": 1.181, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.681, "count": 128, "max_ms": 47.216, "p95_ms": 43.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.463, "count": 128, "max_ms": 98.92, "p95_ms": 86.784}, "yolo11_640_fire_11class": {"avg_ms": 48.924, "count": 128, "max_ms": 76.031, "p95_ms": 65.154}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.028, "count": 128, "max_ms": 94.586, "p95_ms": 81.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.974, "count": 129, "max_ms": 277.68, "p95_ms": 254.731}, "read_interval": {"avg_ms": 233.723, "count": 128, "max_ms": 286.92, "p95_ms": 264.17}, "triton_wait": {"avg_ms": 66.743, "count": 128, "max_ms": 98.933, "p95_ms": 87.885}, "visualize": {"avg_ms": 10.466, "count": 128, "max_ms": 14.439, "p95_ms": 12.666}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 92}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6224.758, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:40:01+09:00", "last_error_summary": "[h264 @ 0x5575c9f88cc0] error while decoding MB 41 19, bytestream -30", "last_exit_at": "2026-07-01T12:40:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.232, "last_started_at": "2026-07-01T12:38:12+09:00", "pid": 5964, "process": {"exists": false, "pid": 5964}, "restart_count": 23, "restart_count_per_hour": 23, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.636, "sample_count": 1256, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11084, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:40:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:40:01+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T12:40:01+09:00", "mjpeg": {"age_sec": 0.229, "exists": true, "mtime": "2026-07-01T12:40:01.632244+09:00", "mtime_epoch": 1782877201.6322439, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188237}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:40:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.805, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.311, "pushed": 4.311, "read": 4.277}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.342, "count": 128, "max_ms": 16.207, "p95_ms": 13.721}, "gstreamer_push": {"avg_ms": 0.865, "count": 129, "max_ms": 1.181, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.681, "count": 128, "max_ms": 47.216, "p95_ms": 43.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.463, "count": 128, "max_ms": 98.92, "p95_ms": 86.784}, "yolo11_640_fire_11class": {"avg_ms": 48.924, "count": 128, "max_ms": 76.031, "p95_ms": 65.154}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.028, "count": 128, "max_ms": 94.586, "p95_ms": 81.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.974, "count": 129, "max_ms": 277.68, "p95_ms": 254.731}, "read_interval": {"avg_ms": 233.723, "count": 128, "max_ms": 286.92, "p95_ms": 264.17}, "triton_wait": {"avg_ms": 66.743, "count": 128, "max_ms": 98.933, "p95_ms": 87.885}, "visualize": {"avg_ms": 10.466, "count": 128, "max_ms": 14.439, "p95_ms": 12.666}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 92}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6325.027, "limit_mb": 8796093022207.996}, "detail": {"pid": 6071, "restart_count": 24, "start_count": 25}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:40:01+09:00", "last_error_summary": "[h264 @ 0x5575c9f88cc0] error while decoding MB 41 19, bytestream -30", "last_exit_at": "2026-07-01T12:40:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:40:04+09:00", "pid": 6071, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.508, "name": "ffmpeg", "pid": 6071, "rss_mb": 136.508, "state": "S (sleeping)", "threads": 106}, "restart_count": 24, "restart_count_per_hour": 24, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.691, "sample_count": 1236, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11084, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:40:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.442, "last_frame_pushed_at": "2026-07-01T12:40:01+09:00", "last_frame_read_age_sec": 4.671, "last_frame_read_at": "2026-07-01T12:40:01+09:00", "mjpeg": {"age_sec": 4.428, "exists": true, "mtime": "2026-07-01T12:40:01.868252+09:00", "mtime_epoch": 1782877201.8682516, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184499}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:40:06+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2355.805, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.648, "pushed": 3.648, "read": 3.615}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.458, "count": 108, "max_ms": 16.207, "p95_ms": 13.996}, "gstreamer_push": {"avg_ms": 0.864, "count": 109, "max_ms": 1.181, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.456, "count": 108, "max_ms": 47.216, "p95_ms": 42.927}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.341, "count": 108, "max_ms": 98.92, "p95_ms": 86.784}, "yolo11_640_fire_11class": {"avg_ms": 48.738, "count": 108, "max_ms": 76.031, "p95_ms": 64.594}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.838, "count": 108, "max_ms": 90.42, "p95_ms": 81.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.905, "count": 109, "max_ms": 277.68, "p95_ms": 256.388}, "read_interval": {"avg_ms": 235.667, "count": 108, "max_ms": 286.92, "p95_ms": 266.172}, "triton_wait": {"avg_ms": 66.587, "count": 108, "max_ms": 98.933, "p95_ms": 87.885}, "visualize": {"avg_ms": 10.55, "count": 108, "max_ms": 14.439, "p95_ms": 12.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_124031_469743e2.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 6259.324, "limit_mb": 8796093022207.996}, "detail": {"pid": 6180, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:40:31+09:00", "pid": 6180, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.41, "name": "ffmpeg", "pid": 6180, "rss_mb": 137.41, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.799}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "469743e2", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.132, "exists": true, "mtime": "2026-07-01T12:40:30.725172+09:00", "mtime_epoch": 1782877230.7251718, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 260800}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T12:40:33+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 703.281, "name": "celery", "pid": 239, "rss_mb": 668.613, "state": "R (running)", "threads": 118}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T12:40:31+09:00", "state": "running", "task_id": "216c73ea-0023-49bf-8ac3-23ab969bd256", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_124140_1d009f23.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 6280.602, "limit_mb": 8796093022207.996}, "detail": {"pid": 6335, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:41:40+09:00", "pid": 6335, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.008, "name": "ffmpeg", "pid": 6335, "rss_mb": 137.008, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.798}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "1d009f23", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.067, "exists": true, "mtime": "2026-07-01T12:41:39.423285+09:00", "mtime_epoch": 1782877299.4232855, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 258589}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T12:41:42+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 703.281, "name": "celery", "pid": 239, "rss_mb": 681.852, "state": "R (running)", "threads": 118}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T12:41:40+09:00", "state": "running", "task_id": "c47417d8-a60d-4e57-bbf6-fa31ec760831", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 96}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6189.758, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:40:01+09:00", "last_error_summary": "[h264 @ 0x5575c9f88cc0] error while decoding MB 41 19, bytestream -30", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:40:01+09:00", "last_error_summary": "[h264 @ 0x5575c9f88cc0] error while decoding MB 41 19, bytestream -30", "last_exit_at": "2026-07-01T12:41:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.469, "last_started_at": "2026-07-01T12:40:04+09:00", "pid": 6071, "process": {"exists": false, "pid": 6071}, "restart_count": 24, "restart_count_per_hour": 24, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.298, "sample_count": 1250, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11547, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:41:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:41:52+09:00", "last_frame_read_age_sec": 0.225, "last_frame_read_at": "2026-07-01T12:41:52+09:00", "mjpeg": {"age_sec": 0.217, "exists": true, "mtime": "2026-07-01T12:41:52.547678+09:00", "mtime_epoch": 1782877312.5476778, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188337}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:41:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.348, "pushed": 4.348, "read": 4.314}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.941, "count": 129, "max_ms": 15.697, "p95_ms": 12.052}, "gstreamer_push": {"avg_ms": 0.874, "count": 130, "max_ms": 1.359, "p95_ms": 0.972}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.237, "count": 129, "max_ms": 45.975, "p95_ms": 41.146}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.644, "count": 129, "max_ms": 99.55, "p95_ms": 95.429}, "yolo11_640_fire_11class": {"avg_ms": 48.901, "count": 129, "max_ms": 70.702, "p95_ms": 61.202}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.733, "count": 129, "max_ms": 92.288, "p95_ms": 78.911}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.828, "count": 130, "max_ms": 286.905, "p95_ms": 261.577}, "read_interval": {"avg_ms": 231.566, "count": 129, "max_ms": 296.162, "p95_ms": 271.044}, "triton_wait": {"avg_ms": 68.609, "count": 129, "max_ms": 99.563, "p95_ms": 95.443}, "visualize": {"avg_ms": 10.237, "count": 129, "max_ms": 16.204, "p95_ms": 11.891}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 96}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6189.75, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:40:01+09:00", "last_error_summary": "[h264 @ 0x5575c9f88cc0] error while decoding MB 41 19, bytestream -30", "last_exit_at": "2026-07-01T12:41:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.472, "last_started_at": "2026-07-01T12:40:04+09:00", "pid": 6071, "process": {"exists": false, "pid": 6071}, "restart_count": 24, "restart_count_per_hour": 24, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.298, "sample_count": 1250, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11547, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:41:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:41:52+09:00", "last_frame_read_age_sec": 0.229, "last_frame_read_at": "2026-07-01T12:41:52+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-01T12:41:52.547678+09:00", "mtime_epoch": 1782877312.5476778, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188337}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:41:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.347, "pushed": 4.347, "read": 4.314}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.941, "count": 129, "max_ms": 15.697, "p95_ms": 12.052}, "gstreamer_push": {"avg_ms": 0.874, "count": 130, "max_ms": 1.359, "p95_ms": 0.972}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.237, "count": 129, "max_ms": 45.975, "p95_ms": 41.146}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.644, "count": 129, "max_ms": 99.55, "p95_ms": 95.429}, "yolo11_640_fire_11class": {"avg_ms": 48.901, "count": 129, "max_ms": 70.702, "p95_ms": 61.202}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.733, "count": 129, "max_ms": 92.288, "p95_ms": 78.911}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.828, "count": 130, "max_ms": 286.905, "p95_ms": 261.577}, "read_interval": {"avg_ms": 231.566, "count": 129, "max_ms": 296.162, "p95_ms": 271.044}, "triton_wait": {"avg_ms": 68.609, "count": 129, "max_ms": 99.563, "p95_ms": 95.443}, "visualize": {"avg_ms": 10.237, "count": 129, "max_ms": 16.204, "p95_ms": 11.891}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 96}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6288.57, "limit_mb": 8796093022207.996}, "detail": {"pid": 6488, "restart_count": 25, "start_count": 26}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:40:01+09:00", "last_error_summary": "[h264 @ 0x5575c9f88cc0] error while decoding MB 41 19, bytestream -30", "last_exit_at": "2026-07-01T12:41:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:41:55+09:00", "pid": 6488, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.672, "name": "ffmpeg", "pid": 6488, "rss_mb": 136.672, "state": "S (sleeping)", "threads": 106}, "restart_count": 25, "restart_count_per_hour": 25, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.369, "sample_count": 1230, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11547, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:41:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.37, "last_frame_pushed_at": "2026-07-01T12:41:52+09:00", "last_frame_read_age_sec": 4.593, "last_frame_read_at": "2026-07-01T12:41:52+09:00", "mjpeg": {"age_sec": 4.352, "exists": true, "mtime": "2026-07-01T12:41:52.779685+09:00", "mtime_epoch": 1782877312.7796845, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188477}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:41:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.062, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.688, "pushed": 3.688, "read": 3.654}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.035, "count": 110, "max_ms": 15.697, "p95_ms": 12.052}, "gstreamer_push": {"avg_ms": 0.872, "count": 110, "max_ms": 1.359, "p95_ms": 0.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.119, "count": 110, "max_ms": 45.975, "p95_ms": 41.06}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.753, "count": 110, "max_ms": 99.55, "p95_ms": 95.751}, "yolo11_640_fire_11class": {"avg_ms": 48.672, "count": 110, "max_ms": 70.702, "p95_ms": 60.842}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.516, "count": 110, "max_ms": 92.288, "p95_ms": 78.911}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.621, "count": 110, "max_ms": 286.905, "p95_ms": 263.196}, "read_interval": {"avg_ms": 233.373, "count": 109, "max_ms": 296.162, "p95_ms": 272.997}, "triton_wait": {"avg_ms": 68.637, "count": 110, "max_ms": 99.563, "p95_ms": 95.763}, "visualize": {"avg_ms": 10.327, "count": 110, "max_ms": 16.204, "p95_ms": 12.203}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.828, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_124242_dd1038fd.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 6355.621, "limit_mb": 8796093022207.996}, "detail": {"pid": 6597, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:42:42+09:00", "pid": 6597, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.723, "name": "ffmpeg", "pid": 6597, "rss_mb": 137.723, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.797}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "dd1038fd", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.173, "exists": true, "mtime": "2026-07-01T12:42:41.309105+09:00", "mtime_epoch": 1782877361.3091054, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 259523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T12:42:44+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 703.281, "name": "celery", "pid": 239, "rss_mb": 686.742, "state": "R (running)", "threads": 118}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T12:42:42+09:00", "state": "running", "task_id": "7b7a6c9f-422f-43c0-89ae-adcf7955af44", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_124243_47051571.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 6364.422, "limit_mb": 8796093022207.996}, "detail": {"pid": 6652, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:42:43+09:00", "pid": 6652, "process": {"exists": true, "fd_count": 5, "hwm_mb": 140.512, "name": "ffmpeg", "pid": 6652, "rss_mb": 140.512, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.744}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "47051571", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 4.335, "exists": true, "mtime": "2026-07-01T12:42:41.309105+09:00", "mtime_epoch": 1782877361.3091054, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 259523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T12:42:45+09:00", "process": {"exists": true, "fd_count": 111, "hwm_mb": 470.484, "name": "celery", "pid": 241, "rss_mb": 470.484, "state": "R (running)", "threads": 7}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T12:42:43+09:00", "state": "running", "task_id": "f71aa48e-c46e-4cea-b30b-555f7ed88ccd", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_124244_fd1cfea1.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 6362.105, "limit_mb": 8796093022207.996}, "detail": {"pid": 6810, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:42:44+09:00", "pid": 6810, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.086, "name": "ffmpeg", "pid": 6810, "rss_mb": 137.086, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.739}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "fd1cfea1", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 5.504, "exists": true, "mtime": "2026-07-01T12:42:41.309105+09:00", "mtime_epoch": 1782877361.3091054, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 259523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T12:42:46+09:00", "process": {"exists": true, "fd_count": 112, "hwm_mb": 470.449, "name": "celery", "pid": 243, "rss_mb": 470.449, "state": "R (running)", "threads": 7}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T12:42:44+09:00", "state": "running", "task_id": "ba4adfb1-3ae7-4ae2-b10f-eee6c6209c0d", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_124245_5314f1c0.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 6302.055, "limit_mb": 8796093022207.996}, "detail": {"pid": 6935, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:42:45+09:00", "pid": 6935, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.594, "name": "ffmpeg", "pid": 6935, "rss_mb": 137.594, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.791}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "5314f1c0", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 6.587, "exists": true, "mtime": "2026-07-01T12:42:41.309105+09:00", "mtime_epoch": 1782877361.3091054, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 259523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T12:42:47+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 703.281, "name": "celery", "pid": 239, "rss_mb": 686.742, "state": "R (running)", "threads": 118}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T12:42:45+09:00", "state": "running", "task_id": "10a7903a-7c1e-4d97-8846-b3c2c9a70951", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6258.102, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55bb9a580d80] error while decoding MB 47 25, bytestream -20", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:43:42+09:00", "last_error_summary": "[h264 @ 0x55bb9a580d80] error while decoding MB 47 25, bytestream -20", "last_exit_at": "2026-07-01T12:41:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.323, "last_started_at": "2026-07-01T12:41:55+09:00", "pid": 6488, "process": {"exists": false, "pid": 6488}, "restart_count": 25, "restart_count_per_hour": 25, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.288, "sample_count": 1268, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12019, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:43:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T12:43:42+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T12:43:42+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T12:43:42.414833+09:00", "mtime_epoch": 1782877422.4148333, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185322}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:43:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.062, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.503, "pushed": 4.503, "read": 4.503}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.627, "count": 134, "max_ms": 14.19, "p95_ms": 11.599}, "gstreamer_push": {"avg_ms": 0.862, "count": 135, "max_ms": 1.279, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.374, "count": 134, "max_ms": 50.592, "p95_ms": 44.199}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.648, "count": 134, "max_ms": 101.674, "p95_ms": 87.275}, "yolo11_640_fire_11class": {"avg_ms": 46.483, "count": 134, "max_ms": 65.372, "p95_ms": 59.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.009, "count": 134, "max_ms": 100.046, "p95_ms": 81.277}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.632, "count": 135, "max_ms": 293.431, "p95_ms": 245.479}, "read_interval": {"avg_ms": 223.356, "count": 135, "max_ms": 303.957, "p95_ms": 254.558}, "triton_wait": {"avg_ms": 63.804, "count": 134, "max_ms": 101.685, "p95_ms": 87.288}, "visualize": {"avg_ms": 10.103, "count": 134, "max_ms": 13.903, "p95_ms": 11.733}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6257.34, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:43:42+09:00", "last_error_summary": "[h264 @ 0x55bb9a580d80] error while decoding MB 47 25, bytestream -20", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:43:42+09:00", "last_error_summary": "[h264 @ 0x55bb9a580d80] error while decoding MB 47 25, bytestream -20", "last_exit_at": "2026-07-01T12:43:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.48, "last_started_at": "2026-07-01T12:41:55+09:00", "pid": 6488, "process": {"exists": false, "pid": 6488}, "restart_count": 25, "restart_count_per_hour": 25, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.288, "sample_count": 1268, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12020, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:43:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T12:43:42+09:00", "last_frame_read_age_sec": 0.208, "last_frame_read_at": "2026-07-01T12:43:42+09:00", "mjpeg": {"age_sec": 0.196, "exists": true, "mtime": "2026-07-01T12:43:42.414833+09:00", "mtime_epoch": 1782877422.4148333, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185322}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:43:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.515, "pushed": 4.515, "read": 4.481}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.632, "count": 134, "max_ms": 14.19, "p95_ms": 11.599}, "gstreamer_push": {"avg_ms": 0.863, "count": 135, "max_ms": 1.279, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.263, "count": 134, "max_ms": 50.592, "p95_ms": 42.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.645, "count": 134, "max_ms": 101.674, "p95_ms": 87.275}, "yolo11_640_fire_11class": {"avg_ms": 46.544, "count": 134, "max_ms": 65.372, "p95_ms": 59.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.027, "count": 134, "max_ms": 100.046, "p95_ms": 81.277}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.457, "count": 135, "max_ms": 293.431, "p95_ms": 245.479}, "read_interval": {"avg_ms": 223.268, "count": 134, "max_ms": 303.957, "p95_ms": 254.558}, "triton_wait": {"avg_ms": 63.822, "count": 134, "max_ms": 101.685, "p95_ms": 87.288}, "visualize": {"avg_ms": 10.09, "count": 134, "max_ms": 13.903, "p95_ms": 11.733}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6257.367, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:43:42+09:00", "last_error_summary": "[h264 @ 0x55bb9a580d80] error while decoding MB 47 25, bytestream -20", "last_exit_at": "2026-07-01T12:43:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.483, "last_started_at": "2026-07-01T12:41:55+09:00", "pid": 6488, "process": {"exists": false, "pid": 6488}, "restart_count": 25, "restart_count_per_hour": 25, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.288, "sample_count": 1268, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12020, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:43:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T12:43:42+09:00", "last_frame_read_age_sec": 0.211, "last_frame_read_at": "2026-07-01T12:43:42+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T12:43:42.414833+09:00", "mtime_epoch": 1782877422.4148333, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185322}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:43:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.514, "pushed": 4.514, "read": 4.481}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.632, "count": 134, "max_ms": 14.19, "p95_ms": 11.599}, "gstreamer_push": {"avg_ms": 0.863, "count": 135, "max_ms": 1.279, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.263, "count": 134, "max_ms": 50.592, "p95_ms": 42.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.645, "count": 134, "max_ms": 101.674, "p95_ms": 87.275}, "yolo11_640_fire_11class": {"avg_ms": 46.544, "count": 134, "max_ms": 65.372, "p95_ms": 59.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.027, "count": 134, "max_ms": 100.046, "p95_ms": 81.277}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.457, "count": 135, "max_ms": 293.431, "p95_ms": 245.479}, "read_interval": {"avg_ms": 223.268, "count": 134, "max_ms": 303.957, "p95_ms": 254.558}, "triton_wait": {"avg_ms": 63.822, "count": 134, "max_ms": 101.685, "p95_ms": 87.288}, "visualize": {"avg_ms": 10.09, "count": 134, "max_ms": 13.903, "p95_ms": 11.733}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_124344_715b4d2a.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 6374.102, "limit_mb": 8796093022207.996}, "detail": {"pid": 7283, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:43:44+09:00", "pid": 7283, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.57, "name": "ffmpeg", "pid": 7283, "rss_mb": 137.57, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.793}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "warning", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["mjpeg_age_sec>10s"]}, "instance_id": "715b4d2a", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 15.313, "exists": true, "mtime": "2026-07-01T12:43:30.838511+09:00", "mtime_epoch": 1782877410.838511, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 259619}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T12:43:46+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 703.297, "name": "celery", "pid": 239, "rss_mb": 684.633, "state": "R (running)", "threads": 118}, "reason_codes": ["mjpeg_output_stale_warning"], "recommended_action": "degrade_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T12:43:43+09:00", "state": "running", "task_id": "680b9148-5bb0-49b6-8499-22aa554e8861", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6469.547, "limit_mb": 8796093022207.996}, "detail": {"pid": 7303, "restart_count": 26, "start_count": 27}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:43:42+09:00", "last_error_summary": "[h264 @ 0x55bb9a580d80] error while decoding MB 47 25, bytestream -20", "last_exit_at": "2026-07-01T12:43:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:43:44+09:00", "pid": 7303, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.262, "name": "ffmpeg", "pid": 7303, "rss_mb": 136.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 26, "restart_count_per_hour": 26, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 251.931, "sample_count": 1250, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12020, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:43:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.248, "last_frame_pushed_at": "2026-07-01T12:43:42+09:00", "last_frame_read_age_sec": 4.451, "last_frame_read_at": "2026-07-01T12:43:42+09:00", "mjpeg": {"age_sec": 4.232, "exists": true, "mtime": "2026-07-01T12:43:42.622839+09:00", "mtime_epoch": 1782877422.6228392, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185652}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:43:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.062, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.886, "pushed": 3.886, "read": 3.853}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.669, "count": 116, "max_ms": 13.069, "p95_ms": 11.599}, "gstreamer_push": {"avg_ms": 0.861, "count": 116, "max_ms": 1.279, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.968, "count": 116, "max_ms": 50.592, "p95_ms": 41.446}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.495, "count": 116, "max_ms": 101.674, "p95_ms": 88.763}, "yolo11_640_fire_11class": {"avg_ms": 46.138, "count": 116, "max_ms": 65.372, "p95_ms": 57.573}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.561, "count": 116, "max_ms": 100.046, "p95_ms": 81.487}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.068, "count": 116, "max_ms": 293.431, "p95_ms": 245.479}, "read_interval": {"avg_ms": 222.866, "count": 115, "max_ms": 303.957, "p95_ms": 254.558}, "triton_wait": {"avg_ms": 63.403, "count": 116, "max_ms": 101.685, "p95_ms": 88.778}, "visualize": {"avg_ms": 10.125, "count": 116, "max_ms": 13.903, "p95_ms": 11.602}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.848, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6195.047, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b75b10e540] error while decoding MB 15 15, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:45:33+09:00", "last_error_summary": "[h264 @ 0x55b75b10e540] error while decoding MB 15 15, bytestream -5", "last_exit_at": "2026-07-01T12:43:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.307, "last_started_at": "2026-07-01T12:43:44+09:00", "pid": 7303, "process": {"exists": false, "pid": 7303}, "restart_count": 26, "restart_count_per_hour": 26, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.479, "sample_count": 1288, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12486, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:45:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T12:45:33+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T12:45:33+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-01T12:45:33.121814+09:00", "mtime_epoch": 1782877533.1218143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184965}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:45:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.32, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.234, "pushed": 4.234, "read": 4.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.297, "count": 127, "max_ms": 17.951, "p95_ms": 12.355}, "gstreamer_push": {"avg_ms": 0.869, "count": 127, "max_ms": 1.166, "p95_ms": 0.956}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.352, "count": 127, "max_ms": 45.59, "p95_ms": 39.703}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.374, "count": 127, "max_ms": 99.326, "p95_ms": 87.244}, "yolo11_640_fire_11class": {"avg_ms": 47.967, "count": 127, "max_ms": 71.698, "p95_ms": 62.622}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.626, "count": 127, "max_ms": 88.344, "p95_ms": 81.096}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.099, "count": 127, "max_ms": 262.666, "p95_ms": 248.031}, "read_interval": {"avg_ms": 235.776, "count": 128, "max_ms": 272.318, "p95_ms": 257.923}, "triton_wait": {"avg_ms": 67.043, "count": 127, "max_ms": 99.338, "p95_ms": 87.446}, "visualize": {"avg_ms": 10.551, "count": 127, "max_ms": 13.444, "p95_ms": 11.74}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6194.137, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:45:33+09:00", "last_error_summary": "[h264 @ 0x55b75b10e540] error while decoding MB 15 15, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:45:33+09:00", "last_error_summary": "[h264 @ 0x55b75b10e540] error while decoding MB 15 15, bytestream -5", "last_exit_at": "2026-07-01T12:45:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.478, "last_started_at": "2026-07-01T12:43:44+09:00", "pid": 7303, "process": {"exists": false, "pid": 7303}, "restart_count": 26, "restart_count_per_hour": 26, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.479, "sample_count": 1288, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12487, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:45:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:45:33+09:00", "last_frame_read_age_sec": 0.22, "last_frame_read_at": "2026-07-01T12:45:33+09:00", "mjpeg": {"age_sec": 0.21, "exists": true, "mtime": "2026-07-01T12:45:33.121814+09:00", "mtime_epoch": 1782877533.1218143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184965}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:45:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.32, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.275, "pushed": 4.275, "read": 4.242}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.285, "count": 127, "max_ms": 17.951, "p95_ms": 12.355}, "gstreamer_push": {"avg_ms": 0.869, "count": 128, "max_ms": 1.166, "p95_ms": 0.956}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.342, "count": 127, "max_ms": 45.59, "p95_ms": 39.703}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.404, "count": 127, "max_ms": 99.326, "p95_ms": 87.244}, "yolo11_640_fire_11class": {"avg_ms": 48.052, "count": 127, "max_ms": 71.698, "p95_ms": 62.622}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.649, "count": 127, "max_ms": 88.344, "p95_ms": 81.096}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.021, "count": 128, "max_ms": 262.666, "p95_ms": 248.031}, "read_interval": {"avg_ms": 235.799, "count": 127, "max_ms": 272.318, "p95_ms": 257.923}, "triton_wait": {"avg_ms": 67.067, "count": 127, "max_ms": 99.338, "p95_ms": 87.446}, "visualize": {"avg_ms": 10.535, "count": 127, "max_ms": 13.444, "p95_ms": 11.738}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6194.129, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:45:33+09:00", "last_error_summary": "[h264 @ 0x55b75b10e540] error while decoding MB 15 15, bytestream -5", "last_exit_at": "2026-07-01T12:45:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.482, "last_started_at": "2026-07-01T12:43:44+09:00", "pid": 7303, "process": {"exists": false, "pid": 7303}, "restart_count": 26, "restart_count_per_hour": 26, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.479, "sample_count": 1288, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12487, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:45:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:45:33+09:00", "last_frame_read_age_sec": 0.224, "last_frame_read_at": "2026-07-01T12:45:33+09:00", "mjpeg": {"age_sec": 0.213, "exists": true, "mtime": "2026-07-01T12:45:33.121814+09:00", "mtime_epoch": 1782877533.1218143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184965}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:45:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.32, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.275, "pushed": 4.275, "read": 4.241}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.285, "count": 127, "max_ms": 17.951, "p95_ms": 12.355}, "gstreamer_push": {"avg_ms": 0.869, "count": 128, "max_ms": 1.166, "p95_ms": 0.956}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.342, "count": 127, "max_ms": 45.59, "p95_ms": 39.703}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.404, "count": 127, "max_ms": 99.326, "p95_ms": 87.244}, "yolo11_640_fire_11class": {"avg_ms": 48.052, "count": 127, "max_ms": 71.698, "p95_ms": 62.622}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.649, "count": 127, "max_ms": 88.344, "p95_ms": 81.096}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.021, "count": 128, "max_ms": 262.666, "p95_ms": 248.031}, "read_interval": {"avg_ms": 235.799, "count": 127, "max_ms": 272.318, "p95_ms": 257.923}, "triton_wait": {"avg_ms": 67.067, "count": 127, "max_ms": 99.338, "p95_ms": 87.446}, "visualize": {"avg_ms": 10.535, "count": 127, "max_ms": 13.444, "p95_ms": 11.738}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6303.355, "limit_mb": 8796093022207.996}, "detail": {"pid": 7543, "restart_count": 27, "start_count": 28}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:45:33+09:00", "last_error_summary": "[h264 @ 0x55b75b10e540] error while decoding MB 15 15, bytestream -5", "last_exit_at": "2026-07-01T12:45:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:45:36+09:00", "pid": 7543, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.754, "name": "ffmpeg", "pid": 7543, "rss_mb": 136.754, "state": "S (sleeping)", "threads": 106}, "restart_count": 27, "restart_count_per_hour": 27, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.479, "sample_count": 1267, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12487, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:45:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.803, "last_frame_pushed_at": "2026-07-01T12:45:33+09:00", "last_frame_read_age_sec": 5.021, "last_frame_read_at": "2026-07-01T12:45:33+09:00", "mjpeg": {"age_sec": 4.786, "exists": true, "mtime": "2026-07-01T12:45:33.345820+09:00", "mtime_epoch": 1782877533.34582, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178341}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:45:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.32, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.58, "pushed": 3.58, "read": 3.546}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.351, "count": 106, "max_ms": 17.951, "p95_ms": 12.355}, "gstreamer_push": {"avg_ms": 0.868, "count": 107, "max_ms": 1.166, "p95_ms": 0.978}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.295, "count": 106, "max_ms": 45.59, "p95_ms": 39.703}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.407, "count": 106, "max_ms": 99.326, "p95_ms": 87.432}, "yolo11_640_fire_11class": {"avg_ms": 47.955, "count": 106, "max_ms": 71.698, "p95_ms": 63.396}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.538, "count": 106, "max_ms": 88.344, "p95_ms": 81.096}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.849, "count": 107, "max_ms": 262.666, "p95_ms": 249.308}, "read_interval": {"avg_ms": 236.627, "count": 106, "max_ms": 272.318, "p95_ms": 258.165}, "triton_wait": {"avg_ms": 67.036, "count": 106, "max_ms": 99.338, "p95_ms": 87.991}, "visualize": {"avg_ms": 10.596, "count": 106, "max_ms": 13.444, "p95_ms": 11.74}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_115104_ab775f64.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 8, "tracking_summary": 270}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6434.086, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 3295.261, "last_started_at": "2026-07-01T11:51:04+09:00", "pid": 2554, "process": {"exists": true, "fd_count": 5, "hwm_mb": 338.309, "name": "ffmpeg", "pid": 2554, "rss_mb": 338.309, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.498, "sample_count": 2489, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32425, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:45:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "ab775f64", "last_frame_pushed_age_sec": 30.056, "last_frame_pushed_at": "2026-07-01T12:45:29+09:00", "last_frame_read_age_sec": 30.141, "last_frame_read_at": "2026-07-01T12:45:29+09:00", "mjpeg": {"age_sec": 30.04, "exists": true, "mtime": "2026-07-01T12:45:29.561721+09:00", "mtime_epoch": 1782877529.561721, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242388}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T12:45:59+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1648.73, "name": "celery", "pid": 267, "rss_mb": 1317.137, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:51:04+09:00", "state": "running", "task_id": "64e6ede9-0997-463c-b342-3af5d237a507", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_115104_ab775f64.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 8, "tracking_summary": 270}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6413.578, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 3295.273, "last_started_at": "2026-07-01T11:51:04+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.498, "sample_count": 2489, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32425, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:45:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "ab775f64", "last_frame_pushed_age_sec": 30.068, "last_frame_pushed_at": "2026-07-01T12:45:29+09:00", "last_frame_read_age_sec": 30.153, "last_frame_read_at": "2026-07-01T12:45:29+09:00", "mjpeg": {"age_sec": 30.052, "exists": true, "mtime": "2026-07-01T12:45:29.561721+09:00", "mtime_epoch": 1782877529.561721, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242388}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T12:45:59+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1648.73, "name": "celery", "pid": 267, "rss_mb": 1317.137, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:51:04+09:00", "state": "running", "task_id": "64e6ede9-0997-463c-b342-3af5d237a507", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_115104_ab775f64.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 8, "tracking_summary": 270}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6124.246, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5589545bbb40] error while decoding MB 73 65, bytestream -29", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:45:59+09:00", "last_error_summary": "[h264 @ 0x5589545bbb40] error while decoding MB 73 65, bytestream -29", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 3295.32, "last_started_at": "2026-07-01T11:51:04+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.498, "sample_count": 2489, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32425, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:45:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "ab775f64", "last_frame_pushed_age_sec": 30.116, "last_frame_pushed_at": "2026-07-01T12:45:29+09:00", "last_frame_read_age_sec": 30.2, "last_frame_read_at": "2026-07-01T12:45:29+09:00", "mjpeg": {"age_sec": 30.099, "exists": true, "mtime": "2026-07-01T12:45:29.561721+09:00", "mtime_epoch": 1782877529.561721, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242388}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T12:45:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1648.73, "name": "celery", "pid": 267, "rss_mb": 1317.137, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:51:04+09:00", "state": "running", "task_id": "64e6ede9-0997-463c-b342-3af5d237a507", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_115104_ab775f64.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 8, "tracking_summary": 270}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6295.895, "limit_mb": 8796093022207.996}, "detail": {"pid": 7650, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:45:59+09:00", "last_error_summary": "[h264 @ 0x5589545bbb40] error while decoding MB 73 65, bytestream -29", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:46:02+09:00", "pid": 7650, "process": {"exists": true, "fd_count": 5, "hwm_mb": 199.098, "name": "ffmpeg", "pid": 7650, "rss_mb": 199.098, "state": "S (sleeping)", "threads": 106}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 161.897, "sample_count": 2440, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32425, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:45:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "ab775f64", "last_frame_pushed_age_sec": 35.013, "last_frame_pushed_at": "2026-07-01T12:45:29+09:00", "last_frame_read_age_sec": 35.097, "last_frame_read_at": "2026-07-01T12:45:29+09:00", "mjpeg": {"age_sec": 34.996, "exists": true, "mtime": "2026-07-01T12:45:29.561721+09:00", "mtime_epoch": 1782877529.561721, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242388}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T12:46:04+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1648.73, "name": "celery", "pid": 267, "rss_mb": 1317.141, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:51:04+09:00", "state": "running", "task_id": "64e6ede9-0997-463c-b342-3af5d237a507", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 108}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5973.465, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f167207180] error while decoding MB 70 30, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:47:42+09:00", "last_error_summary": "[h264 @ 0x55f167207180] error while decoding MB 70 30, bytestream -6", "last_exit_at": "2026-07-01T12:45:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.992, "last_started_at": "2026-07-01T12:45:36+09:00", "pid": 7543, "process": {"exists": false, "pid": 7543}, "restart_count": 27, "restart_count_per_hour": 27, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.396, "sample_count": 1298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13046, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:47:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T12:47:42+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T12:47:42+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T12:47:42.077085+09:00", "mtime_epoch": 1782877662.0770853, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:47:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.578, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.492, "pushed": 4.492, "read": 4.492}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.744, "count": 134, "max_ms": 15.156, "p95_ms": 11.475}, "gstreamer_push": {"avg_ms": 0.854, "count": 134, "max_ms": 1.515, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.932, "count": 134, "max_ms": 47.794, "p95_ms": 42.598}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.43, "count": 134, "max_ms": 94.348, "p95_ms": 84.557}, "yolo11_640_fire_11class": {"avg_ms": 46.872, "count": 134, "max_ms": 66.445, "p95_ms": 60.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.962, "count": 134, "max_ms": 92.669, "p95_ms": 80.558}}, "truncated_count": 0}, "processing_total": {"avg_ms": 214.138, "count": 134, "max_ms": 252.797, "p95_ms": 239.359}, "read_interval": {"avg_ms": 223.909, "count": 134, "max_ms": 262.764, "p95_ms": 250.156}, "triton_wait": {"avg_ms": 64.649, "count": 134, "max_ms": 94.361, "p95_ms": 85.299}, "visualize": {"avg_ms": 10.123, "count": 134, "max_ms": 14.202, "p95_ms": 11.77}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.83, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 108}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5972.766, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:47:42+09:00", "last_error_summary": "[h264 @ 0x55f167207180] error while decoding MB 70 30, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:47:42+09:00", "last_error_summary": "[h264 @ 0x55f167207180] error while decoding MB 70 30, bytestream -6", "last_exit_at": "2026-07-01T12:47:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.147, "last_started_at": "2026-07-01T12:45:36+09:00", "pid": 7543, "process": {"exists": false, "pid": 7543}, "restart_count": 27, "restart_count_per_hour": 27, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.396, "sample_count": 1298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13047, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:47:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:47:42+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T12:47:42+09:00", "mjpeg": {"age_sec": 0.201, "exists": true, "mtime": "2026-07-01T12:47:42.077085+09:00", "mtime_epoch": 1782877662.0770853, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:47:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.578, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.502, "pushed": 4.502, "read": 4.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.75, "count": 134, "max_ms": 15.156, "p95_ms": 11.475}, "gstreamer_push": {"avg_ms": 0.854, "count": 135, "max_ms": 1.515, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.923, "count": 134, "max_ms": 47.794, "p95_ms": 42.598}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.377, "count": 134, "max_ms": 94.348, "p95_ms": 84.557}, "yolo11_640_fire_11class": {"avg_ms": 46.834, "count": 134, "max_ms": 66.445, "p95_ms": 60.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.919, "count": 134, "max_ms": 92.669, "p95_ms": 80.558}}, "truncated_count": 0}, "processing_total": {"avg_ms": 214.074, "count": 135, "max_ms": 252.797, "p95_ms": 239.359}, "read_interval": {"avg_ms": 223.909, "count": 134, "max_ms": 262.764, "p95_ms": 250.156}, "triton_wait": {"avg_ms": 64.607, "count": 134, "max_ms": 94.361, "p95_ms": 85.299}, "visualize": {"avg_ms": 10.119, "count": 134, "max_ms": 14.202, "p95_ms": 11.77}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 108}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5972.883, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:47:42+09:00", "last_error_summary": "[h264 @ 0x55f167207180] error while decoding MB 70 30, bytestream -6", "last_exit_at": "2026-07-01T12:47:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.151, "last_started_at": "2026-07-01T12:45:36+09:00", "pid": 7543, "process": {"exists": false, "pid": 7543}, "restart_count": 27, "restart_count_per_hour": 27, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.396, "sample_count": 1298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13047, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:47:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T12:47:42+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T12:47:42+09:00", "mjpeg": {"age_sec": 0.205, "exists": true, "mtime": "2026-07-01T12:47:42.077085+09:00", "mtime_epoch": 1782877662.0770853, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:47:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.578, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.502, "pushed": 4.502, "read": 4.468}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.75, "count": 134, "max_ms": 15.156, "p95_ms": 11.475}, "gstreamer_push": {"avg_ms": 0.854, "count": 135, "max_ms": 1.515, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.923, "count": 134, "max_ms": 47.794, "p95_ms": 42.598}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.377, "count": 134, "max_ms": 94.348, "p95_ms": 84.557}, "yolo11_640_fire_11class": {"avg_ms": 46.834, "count": 134, "max_ms": 66.445, "p95_ms": 60.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.919, "count": 134, "max_ms": 92.669, "p95_ms": 80.558}}, "truncated_count": 0}, "processing_total": {"avg_ms": 214.074, "count": 135, "max_ms": 252.797, "p95_ms": 239.359}, "read_interval": {"avg_ms": 223.909, "count": 134, "max_ms": 262.764, "p95_ms": 250.156}, "triton_wait": {"avg_ms": 64.607, "count": 134, "max_ms": 94.361, "p95_ms": 85.299}, "visualize": {"avg_ms": 10.119, "count": 134, "max_ms": 14.202, "p95_ms": 11.77}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 108}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6078.242, "limit_mb": 8796093022207.996}, "detail": {"pid": 7862, "restart_count": 28, "start_count": 29}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:47:42+09:00", "last_error_summary": "[h264 @ 0x55f167207180] error while decoding MB 70 30, bytestream -6", "last_exit_at": "2026-07-01T12:47:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:47:44+09:00", "pid": 7862, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.617, "name": "ffmpeg", "pid": 7862, "rss_mb": 136.617, "state": "S (sleeping)", "threads": 106}, "restart_count": 28, "restart_count_per_hour": 28, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.546, "sample_count": 1278, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13047, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:47:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.148, "last_frame_pushed_at": "2026-07-01T12:47:42+09:00", "last_frame_read_age_sec": 4.355, "last_frame_read_at": "2026-07-01T12:47:42+09:00", "mjpeg": {"age_sec": 4.134, "exists": true, "mtime": "2026-07-01T12:47:42.289091+09:00", "mtime_epoch": 1782877662.2890906, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187183}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:47:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.578, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.918, "pushed": 3.918, "read": 3.885}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.635, "count": 116, "max_ms": 15.156, "p95_ms": 11.475}, "gstreamer_push": {"avg_ms": 0.856, "count": 117, "max_ms": 1.515, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.157, "count": 116, "max_ms": 47.794, "p95_ms": 43.401}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.353, "count": 116, "max_ms": 94.348, "p95_ms": 85.287}, "yolo11_640_fire_11class": {"avg_ms": 47.281, "count": 116, "max_ms": 66.445, "p95_ms": 60.89}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.033, "count": 116, "max_ms": 92.669, "p95_ms": 80.885}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.929, "count": 117, "max_ms": 252.797, "p95_ms": 238.126}, "read_interval": {"avg_ms": 221.784, "count": 116, "max_ms": 262.764, "p95_ms": 248.805}, "triton_wait": {"avg_ms": 64.655, "count": 116, "max_ms": 94.361, "p95_ms": 86.158}, "visualize": {"avg_ms": 10.029, "count": 116, "max_ms": 14.202, "p95_ms": 11.92}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.86, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5974.055, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c97a513480] error while decoding MB 66 35, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:49:51+09:00", "last_error_summary": "[h264 @ 0x55c97a513480] error while decoding MB 66 35, bytestream -10", "last_exit_at": "2026-07-01T12:47:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.028, "last_started_at": "2026-07-01T12:47:44+09:00", "pid": 7862, "process": {"exists": false, "pid": 7862}, "restart_count": 28, "restart_count_per_hour": 28, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 240.253, "sample_count": 1317, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13627, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:49:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T12:49:51+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T12:49:51+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T12:49:51.408195+09:00", "mtime_epoch": 1782877791.4081953, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200755}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:49:51+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.836, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.713, "pushed": 4.713, "read": 4.713}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.202, "count": 140, "max_ms": 11.609, "p95_ms": 10.596}, "gstreamer_push": {"avg_ms": 0.841, "count": 141, "max_ms": 1.303, "p95_ms": 0.875}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.72, "count": 140, "max_ms": 47.633, "p95_ms": 42.338}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.738, "count": 140, "max_ms": 81.038, "p95_ms": 73.164}, "yolo11_640_fire_11class": {"avg_ms": 47.45, "count": 140, "max_ms": 62.707, "p95_ms": 58.517}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.079, "count": 140, "max_ms": 77.917, "p95_ms": 73.136}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.564, "count": 141, "max_ms": 235.749, "p95_ms": 217.513}, "read_interval": {"avg_ms": 213.195, "count": 141, "max_ms": 246.141, "p95_ms": 226.885}, "triton_wait": {"avg_ms": 61.508, "count": 140, "max_ms": 81.05, "p95_ms": 73.914}, "visualize": {"avg_ms": 9.804, "count": 140, "max_ms": 12.466, "p95_ms": 11.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5973.781, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:49:51+09:00", "last_error_summary": "[h264 @ 0x55c97a513480] error while decoding MB 66 35, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:49:51+09:00", "last_error_summary": "[h264 @ 0x55c97a513480] error while decoding MB 66 35, bytestream -10", "last_exit_at": "2026-07-01T12:49:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.189, "last_started_at": "2026-07-01T12:47:44+09:00", "pid": 7862, "process": {"exists": false, "pid": 7862}, "restart_count": 28, "restart_count_per_hour": 28, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 240.253, "sample_count": 1317, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13628, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:49:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T12:49:51+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T12:49:51+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-01T12:49:51.408195+09:00", "mtime_epoch": 1782877791.4081953, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200755}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:49:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.836, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.721, "pushed": 4.721, "read": 4.687}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.202, "count": 141, "max_ms": 11.609, "p95_ms": 10.596}, "gstreamer_push": {"avg_ms": 0.842, "count": 141, "max_ms": 1.303, "p95_ms": 0.875}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.715, "count": 141, "max_ms": 47.633, "p95_ms": 42.338}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.732, "count": 141, "max_ms": 81.038, "p95_ms": 73.164}, "yolo11_640_fire_11class": {"avg_ms": 47.488, "count": 141, "max_ms": 62.707, "p95_ms": 58.517}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.079, "count": 141, "max_ms": 77.917, "p95_ms": 73.136}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.628, "count": 141, "max_ms": 235.749, "p95_ms": 217.513}, "read_interval": {"avg_ms": 213.268, "count": 140, "max_ms": 246.141, "p95_ms": 226.885}, "triton_wait": {"avg_ms": 61.506, "count": 141, "max_ms": 81.05, "p95_ms": 73.914}, "visualize": {"avg_ms": 9.804, "count": 141, "max_ms": 12.466, "p95_ms": 11.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5973.898, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:49:51+09:00", "last_error_summary": "[h264 @ 0x55c97a513480] error while decoding MB 66 35, bytestream -10", "last_exit_at": "2026-07-01T12:49:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.192, "last_started_at": "2026-07-01T12:47:44+09:00", "pid": 7862, "process": {"exists": false, "pid": 7862}, "restart_count": 28, "restart_count_per_hour": 28, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 240.253, "sample_count": 1317, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13628, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:49:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T12:49:51+09:00", "last_frame_read_age_sec": 0.213, "last_frame_read_at": "2026-07-01T12:49:51+09:00", "mjpeg": {"age_sec": 0.206, "exists": true, "mtime": "2026-07-01T12:49:51.408195+09:00", "mtime_epoch": 1782877791.4081953, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200755}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:49:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.836, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.72, "pushed": 4.72, "read": 4.687}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.202, "count": 141, "max_ms": 11.609, "p95_ms": 10.596}, "gstreamer_push": {"avg_ms": 0.842, "count": 141, "max_ms": 1.303, "p95_ms": 0.875}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.715, "count": 141, "max_ms": 47.633, "p95_ms": 42.338}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.732, "count": 141, "max_ms": 81.038, "p95_ms": 73.164}, "yolo11_640_fire_11class": {"avg_ms": 47.488, "count": 141, "max_ms": 62.707, "p95_ms": 58.517}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.079, "count": 141, "max_ms": 77.917, "p95_ms": 73.136}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.628, "count": 141, "max_ms": 235.749, "p95_ms": 217.513}, "read_interval": {"avg_ms": 213.268, "count": 140, "max_ms": 246.141, "p95_ms": 226.885}, "triton_wait": {"avg_ms": 61.506, "count": 141, "max_ms": 81.05, "p95_ms": 73.914}, "visualize": {"avg_ms": 9.804, "count": 141, "max_ms": 12.466, "p95_ms": 11.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 113}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6081.695, "limit_mb": 8796093022207.996}, "detail": {"pid": 7969, "restart_count": 29, "start_count": 30}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:49:51+09:00", "last_error_summary": "[h264 @ 0x55c97a513480] error while decoding MB 66 35, bytestream -10", "last_exit_at": "2026-07-01T12:49:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:49:54+09:00", "pid": 7969, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.895, "name": "ffmpeg", "pid": 7969, "rss_mb": 138.895, "state": "S (sleeping)", "threads": 106}, "restart_count": 29, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 239.733, "sample_count": 1298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13628, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:49:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.436, "last_frame_pushed_at": "2026-07-01T12:49:51+09:00", "last_frame_read_age_sec": 4.642, "last_frame_read_at": "2026-07-01T12:49:51+09:00", "mjpeg": {"age_sec": 4.423, "exists": true, "mtime": "2026-07-01T12:49:51.620200+09:00", "mtime_epoch": 1782877791.6202002, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201060}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:49:56+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2356.836, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.017, "pushed": 4.017, "read": 3.984}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.22, "count": 120, "max_ms": 11.609, "p95_ms": 10.665}, "gstreamer_push": {"avg_ms": 0.838, "count": 120, "max_ms": 0.886, "p95_ms": 0.875}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.533, "count": 120, "max_ms": 47.633, "p95_ms": 41.851}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.834, "count": 120, "max_ms": 81.038, "p95_ms": 73.199}, "yolo11_640_fire_11class": {"avg_ms": 47.582, "count": 120, "max_ms": 62.707, "p95_ms": 58.169}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.394, "count": 120, "max_ms": 77.917, "p95_ms": 73.481}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.984, "count": 120, "max_ms": 235.749, "p95_ms": 218.0}, "read_interval": {"avg_ms": 213.704, "count": 119, "max_ms": 246.141, "p95_ms": 228.075}, "triton_wait": {"avg_ms": 61.808, "count": 120, "max_ms": 81.05, "p95_ms": 74.269}, "visualize": {"avg_ms": 9.803, "count": 120, "max_ms": 12.466, "p95_ms": 11.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6293.605, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564f5c635c40] error while decoding MB 74 25, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:51:41+09:00", "last_error_summary": "[h264 @ 0x564f5c635c40] error while decoding MB 74 25, bytestream -9", "last_exit_at": "2026-07-01T12:49:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.327, "last_started_at": "2026-07-01T12:49:54+09:00", "pid": 7969, "process": {"exists": false, "pid": 7969}, "restart_count": 29, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.828, "sample_count": 1335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14107, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:51:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T12:51:41+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T12:51:41+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-01T12:51:41.330733+09:00", "mtime_epoch": 1782877901.3307328, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189154}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:51:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.094, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.644, "pushed": 4.644, "read": 4.644}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.367, "count": 138, "max_ms": 11.303, "p95_ms": 10.565}, "gstreamer_push": {"avg_ms": 0.853, "count": 139, "max_ms": 0.923, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.51, "count": 138, "max_ms": 49.634, "p95_ms": 43.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.434, "count": 138, "max_ms": 74.331, "p95_ms": 70.956}, "yolo11_640_fire_11class": {"avg_ms": 47.006, "count": 138, "max_ms": 64.279, "p95_ms": 57.725}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.774, "count": 138, "max_ms": 82.481, "p95_ms": 75.394}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.638, "count": 139, "max_ms": 244.434, "p95_ms": 225.091}, "read_interval": {"avg_ms": 216.428, "count": 139, "max_ms": 253.919, "p95_ms": 234.355}, "triton_wait": {"avg_ms": 61.28, "count": 138, "max_ms": 82.493, "p95_ms": 75.405}, "visualize": {"avg_ms": 9.88, "count": 138, "max_ms": 13.109, "p95_ms": 10.877}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6292.621, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:51:41+09:00", "last_error_summary": "[h264 @ 0x564f5c635c40] error while decoding MB 74 25, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:51:41+09:00", "last_error_summary": "[h264 @ 0x564f5c635c40] error while decoding MB 74 25, bytestream -9", "last_exit_at": "2026-07-01T12:51:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.53, "last_started_at": "2026-07-01T12:49:54+09:00", "pid": 7969, "process": {"exists": false, "pid": 7969}, "restart_count": 29, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.859, "sample_count": 1335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14108, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:51:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:51:41+09:00", "last_frame_read_age_sec": 0.252, "last_frame_read_at": "2026-07-01T12:51:41+09:00", "mjpeg": {"age_sec": 0.241, "exists": true, "mtime": "2026-07-01T12:51:41.330733+09:00", "mtime_epoch": 1782877901.3307328, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189154}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:51:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.094, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.646, "pushed": 4.646, "read": 4.613}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.365, "count": 138, "max_ms": 11.303, "p95_ms": 10.565}, "gstreamer_push": {"avg_ms": 0.852, "count": 139, "max_ms": 0.923, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.497, "count": 138, "max_ms": 49.634, "p95_ms": 43.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.54, "count": 138, "max_ms": 77.618, "p95_ms": 72.326}, "yolo11_640_fire_11class": {"avg_ms": 47.067, "count": 138, "max_ms": 64.279, "p95_ms": 57.876}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.931, "count": 138, "max_ms": 83.641, "p95_ms": 75.666}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.965, "count": 139, "max_ms": 247.396, "p95_ms": 225.161}, "read_interval": {"avg_ms": 216.464, "count": 138, "max_ms": 253.919, "p95_ms": 234.355}, "triton_wait": {"avg_ms": 61.43, "count": 138, "max_ms": 83.653, "p95_ms": 75.679}, "visualize": {"avg_ms": 9.893, "count": 138, "max_ms": 13.109, "p95_ms": 11.006}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6292.613, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:51:41+09:00", "last_error_summary": "[h264 @ 0x564f5c635c40] error while decoding MB 74 25, bytestream -9", "last_exit_at": "2026-07-01T12:51:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.534, "last_started_at": "2026-07-01T12:49:54+09:00", "pid": 7969, "process": {"exists": false, "pid": 7969}, "restart_count": 29, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.859, "sample_count": 1335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14108, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:51:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:51:41+09:00", "last_frame_read_age_sec": 0.255, "last_frame_read_at": "2026-07-01T12:51:41+09:00", "mjpeg": {"age_sec": 0.245, "exists": true, "mtime": "2026-07-01T12:51:41.330733+09:00", "mtime_epoch": 1782877901.3307328, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189154}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:51:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.094, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.646, "pushed": 4.646, "read": 4.612}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.365, "count": 138, "max_ms": 11.303, "p95_ms": 10.565}, "gstreamer_push": {"avg_ms": 0.852, "count": 139, "max_ms": 0.923, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.497, "count": 138, "max_ms": 49.634, "p95_ms": 43.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.54, "count": 138, "max_ms": 77.618, "p95_ms": 72.326}, "yolo11_640_fire_11class": {"avg_ms": 47.067, "count": 138, "max_ms": 64.279, "p95_ms": 57.876}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.931, "count": 138, "max_ms": 83.641, "p95_ms": 75.666}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.965, "count": 139, "max_ms": 247.396, "p95_ms": 225.161}, "read_interval": {"avg_ms": 216.464, "count": 138, "max_ms": 253.919, "p95_ms": 234.355}, "triton_wait": {"avg_ms": 61.43, "count": 138, "max_ms": 83.653, "p95_ms": 75.679}, "visualize": {"avg_ms": 9.893, "count": 138, "max_ms": 13.109, "p95_ms": 11.006}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6179.855, "limit_mb": 8796093022207.996}, "detail": {"pid": 8136, "restart_count": 30, "start_count": 31}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:51:41+09:00", "last_error_summary": "[h264 @ 0x564f5c635c40] error while decoding MB 74 25, bytestream -9", "last_exit_at": "2026-07-01T12:51:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:51:43+09:00", "pid": 8136, "process": {"exists": true, "fd_count": 5, "hwm_mb": 90.387, "name": "ffmpeg", "pid": 8136, "rss_mb": 90.387, "state": "S (sleeping)", "threads": 19}, "restart_count": 30, "restart_count_per_hour": 30, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.707, "sample_count": 1317, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14108, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:51:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.045, "last_frame_pushed_at": "2026-07-01T12:51:41+09:00", "last_frame_read_age_sec": 4.294, "last_frame_read_at": "2026-07-01T12:51:41+09:00", "mjpeg": {"age_sec": 4.027, "exists": true, "mtime": "2026-07-01T12:51:41.586739+09:00", "mtime_epoch": 1782877901.5867386, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:51:45+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.094, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.017, "pushed": 4.017, "read": 3.984}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.37, "count": 120, "max_ms": 11.303, "p95_ms": 10.496}, "gstreamer_push": {"avg_ms": 0.851, "count": 120, "max_ms": 0.923, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.714, "count": 120, "max_ms": 49.634, "p95_ms": 43.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.391, "count": 120, "max_ms": 77.618, "p95_ms": 72.326}, "yolo11_640_fire_11class": {"avg_ms": 47.196, "count": 120, "max_ms": 64.279, "p95_ms": 57.876}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.027, "count": 120, "max_ms": 83.641, "p95_ms": 75.666}}, "truncated_count": 0}, "processing_total": {"avg_ms": 207.16, "count": 120, "max_ms": 247.396, "p95_ms": 225.161}, "read_interval": {"avg_ms": 216.608, "count": 119, "max_ms": 253.919, "p95_ms": 234.704}, "triton_wait": {"avg_ms": 61.461, "count": 120, "max_ms": 83.653, "p95_ms": 75.679}, "visualize": {"avg_ms": 9.905, "count": 120, "max_ms": 13.109, "p95_ms": 10.877}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 121}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5984.934, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560749900ec0] error while decoding MB 60 18, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:53:33+09:00", "last_error_summary": "[h264 @ 0x560749900ec0] error while decoding MB 60 18, bytestream -5", "last_exit_at": "2026-07-01T12:51:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.685, "last_started_at": "2026-07-01T12:51:43+09:00", "pid": 8136, "process": {"exists": false, "pid": 8136}, "restart_count": 30, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.599, "sample_count": 1323, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14584, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:53:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T12:53:33+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T12:53:33+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T12:53:33.253237+09:00", "mtime_epoch": 1782878013.253237, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:53:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.352, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.326, "pushed": 4.326, "read": 4.326}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.131, "count": 129, "max_ms": 14.342, "p95_ms": 11.05}, "gstreamer_push": {"avg_ms": 0.848, "count": 129, "max_ms": 1.261, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.66, "count": 129, "max_ms": 48.447, "p95_ms": 42.419}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.701, "count": 129, "max_ms": 80.137, "p95_ms": 78.433}, "yolo11_640_fire_11class": {"avg_ms": 47.066, "count": 129, "max_ms": 65.011, "p95_ms": 57.837}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.531, "count": 129, "max_ms": 79.345, "p95_ms": 76.722}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.688, "count": 129, "max_ms": 264.068, "p95_ms": 251.614}, "read_interval": {"avg_ms": 232.3, "count": 129, "max_ms": 273.425, "p95_ms": 262.328}, "triton_wait": {"avg_ms": 63.324, "count": 129, "max_ms": 80.148, "p95_ms": 79.165}, "visualize": {"avg_ms": 10.565, "count": 129, "max_ms": 13.852, "p95_ms": 12.102}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.816, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 121}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5983.773, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:53:33+09:00", "last_error_summary": "[h264 @ 0x560749900ec0] error while decoding MB 60 18, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:53:33+09:00", "last_error_summary": "[h264 @ 0x560749900ec0] error while decoding MB 60 18, bytestream -5", "last_exit_at": "2026-07-01T12:53:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.878, "last_started_at": "2026-07-01T12:51:43+09:00", "pid": 8136, "process": {"exists": false, "pid": 8136}, "restart_count": 30, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.604, "sample_count": 1323, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:53:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:53:33+09:00", "last_frame_read_age_sec": 0.246, "last_frame_read_at": "2026-07-01T12:53:33+09:00", "mjpeg": {"age_sec": 0.238, "exists": true, "mtime": "2026-07-01T12:53:33.253237+09:00", "mtime_epoch": 1782878013.253237, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:53:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.352, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.328, "pushed": 4.328, "read": 4.295}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.138, "count": 129, "max_ms": 14.342, "p95_ms": 11.05}, "gstreamer_push": {"avg_ms": 0.848, "count": 129, "max_ms": 1.261, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.656, "count": 129, "max_ms": 48.447, "p95_ms": 42.419}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.871, "count": 129, "max_ms": 80.137, "p95_ms": 78.433}, "yolo11_640_fire_11class": {"avg_ms": 47.193, "count": 129, "max_ms": 65.011, "p95_ms": 58.83}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.666, "count": 129, "max_ms": 79.345, "p95_ms": 76.722}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.99, "count": 129, "max_ms": 264.068, "p95_ms": 251.614}, "read_interval": {"avg_ms": 232.459, "count": 128, "max_ms": 273.425, "p95_ms": 262.328}, "triton_wait": {"avg_ms": 63.47, "count": 129, "max_ms": 80.148, "p95_ms": 79.165}, "visualize": {"avg_ms": 10.572, "count": 129, "max_ms": 13.852, "p95_ms": 12.102}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.803, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 121}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5983.766, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:53:33+09:00", "last_error_summary": "[h264 @ 0x560749900ec0] error while decoding MB 60 18, bytestream -5", "last_exit_at": "2026-07-01T12:53:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.882, "last_started_at": "2026-07-01T12:51:43+09:00", "pid": 8136, "process": {"exists": false, "pid": 8136}, "restart_count": 30, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.604, "sample_count": 1323, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:53:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:53:33+09:00", "last_frame_read_age_sec": 0.249, "last_frame_read_at": "2026-07-01T12:53:33+09:00", "mjpeg": {"age_sec": 0.242, "exists": true, "mtime": "2026-07-01T12:53:33.253237+09:00", "mtime_epoch": 1782878013.253237, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:53:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.352, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.328, "pushed": 4.328, "read": 4.294}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.138, "count": 129, "max_ms": 14.342, "p95_ms": 11.05}, "gstreamer_push": {"avg_ms": 0.848, "count": 129, "max_ms": 1.261, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.656, "count": 129, "max_ms": 48.447, "p95_ms": 42.419}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.871, "count": 129, "max_ms": 80.137, "p95_ms": 78.433}, "yolo11_640_fire_11class": {"avg_ms": 47.193, "count": 129, "max_ms": 65.011, "p95_ms": 58.83}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.666, "count": 129, "max_ms": 79.345, "p95_ms": 76.722}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.99, "count": 129, "max_ms": 264.068, "p95_ms": 251.614}, "read_interval": {"avg_ms": 232.459, "count": 128, "max_ms": 273.425, "p95_ms": 262.328}, "triton_wait": {"avg_ms": 63.47, "count": 129, "max_ms": 80.148, "p95_ms": 79.165}, "visualize": {"avg_ms": 10.572, "count": 129, "max_ms": 13.852, "p95_ms": 12.102}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.807, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 121}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6090.293, "limit_mb": 8796093022207.996}, "detail": {"pid": 8243, "restart_count": 31, "start_count": 32}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:53:33+09:00", "last_error_summary": "[h264 @ 0x560749900ec0] error while decoding MB 60 18, bytestream -5", "last_exit_at": "2026-07-01T12:53:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:53:36+09:00", "pid": 8243, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.535, "name": "ffmpeg", "pid": 8243, "rss_mb": 136.535, "state": "S (sleeping)", "threads": 106}, "restart_count": 31, "restart_count_per_hour": 30, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.638, "sample_count": 1301, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14585, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:53:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.775, "last_frame_pushed_at": "2026-07-01T12:53:33+09:00", "last_frame_read_age_sec": 5.017, "last_frame_read_at": "2026-07-01T12:53:33+09:00", "mjpeg": {"age_sec": 4.758, "exists": true, "mtime": "2026-07-01T12:53:33.505243+09:00", "mtime_epoch": 1782878013.5052426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188748}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:53:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.352, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.58, "pushed": 3.58, "read": 3.546}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.348, "count": 106, "max_ms": 14.342, "p95_ms": 11.152}, "gstreamer_push": {"avg_ms": 0.849, "count": 107, "max_ms": 1.261, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.492, "count": 106, "max_ms": 48.447, "p95_ms": 41.267}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.181, "count": 106, "max_ms": 80.137, "p95_ms": 78.905}, "yolo11_640_fire_11class": {"avg_ms": 46.963, "count": 106, "max_ms": 62.112, "p95_ms": 58.83}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.005, "count": 106, "max_ms": 79.345, "p95_ms": 76.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.118, "count": 107, "max_ms": 264.068, "p95_ms": 253.542}, "read_interval": {"avg_ms": 236.58, "count": 106, "max_ms": 273.425, "p95_ms": 262.716}, "triton_wait": {"avg_ms": 63.832, "count": 106, "max_ms": 80.148, "p95_ms": 79.294}, "visualize": {"avg_ms": 10.776, "count": 106, "max_ms": 13.852, "p95_ms": 12.18}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5986.246, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5649dc71c780] error while decoding MB 56 18, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:55:43+09:00", "last_error_summary": "[h264 @ 0x5649dc71c780] error while decoding MB 56 18, bytestream -7", "last_exit_at": "2026-07-01T12:53:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.184, "last_started_at": "2026-07-01T12:53:36+09:00", "pid": 8243, "process": {"exists": false, "pid": 8243}, "restart_count": 31, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.934, "sample_count": 1331, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15169, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:55:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T12:55:43+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T12:55:43+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-01T12:55:43.396069+09:00", "mtime_epoch": 1782878143.396069, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181780}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:55:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.609, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.694, "pushed": 4.694, "read": 4.694}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.187, "count": 140, "max_ms": 14.257, "p95_ms": 10.215}, "gstreamer_push": {"avg_ms": 0.847, "count": 140, "max_ms": 1.007, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.196, "count": 140, "max_ms": 49.138, "p95_ms": 36.448}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.748, "count": 140, "max_ms": 80.793, "p95_ms": 73.59}, "yolo11_640_fire_11class": {"avg_ms": 47.525, "count": 140, "max_ms": 66.894, "p95_ms": 58.208}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.858, "count": 140, "max_ms": 83.5, "p95_ms": 73.826}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.427, "count": 140, "max_ms": 235.549, "p95_ms": 219.097}, "read_interval": {"avg_ms": 214.108, "count": 140, "max_ms": 245.098, "p95_ms": 228.856}, "triton_wait": {"avg_ms": 61.546, "count": 140, "max_ms": 83.513, "p95_ms": 75.363}, "visualize": {"avg_ms": 9.66, "count": 140, "max_ms": 11.168, "p95_ms": 10.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.828, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5985.762, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:55:43+09:00", "last_error_summary": "[h264 @ 0x5649dc71c780] error while decoding MB 56 18, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:55:43+09:00", "last_error_summary": "[h264 @ 0x5649dc71c780] error while decoding MB 56 18, bytestream -7", "last_exit_at": "2026-07-01T12:55:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.356, "last_started_at": "2026-07-01T12:53:36+09:00", "pid": 8243, "process": {"exists": false, "pid": 8243}, "restart_count": 31, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.878, "sample_count": 1332, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15170, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:55:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:55:43+09:00", "last_frame_read_age_sec": 0.227, "last_frame_read_at": "2026-07-01T12:55:43+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T12:55:43.396069+09:00", "mtime_epoch": 1782878143.396069, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181780}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:55:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.609, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.667, "pushed": 4.7, "read": 4.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.188, "count": 140, "max_ms": 14.257, "p95_ms": 10.215}, "gstreamer_push": {"avg_ms": 0.847, "count": 141, "max_ms": 1.007, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.197, "count": 140, "max_ms": 49.138, "p95_ms": 36.448}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.866, "count": 140, "max_ms": 80.793, "p95_ms": 73.972}, "yolo11_640_fire_11class": {"avg_ms": 47.575, "count": 140, "max_ms": 66.894, "p95_ms": 58.208}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.003, "count": 140, "max_ms": 83.5, "p95_ms": 75.045}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.575, "count": 140, "max_ms": 235.549, "p95_ms": 219.359}, "read_interval": {"avg_ms": 214.108, "count": 140, "max_ms": 245.098, "p95_ms": 228.856}, "triton_wait": {"avg_ms": 61.691, "count": 140, "max_ms": 83.513, "p95_ms": 76.127}, "visualize": {"avg_ms": 9.66, "count": 140, "max_ms": 11.168, "p95_ms": 10.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5985.754, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:55:43+09:00", "last_error_summary": "[h264 @ 0x5649dc71c780] error while decoding MB 56 18, bytestream -7", "last_exit_at": "2026-07-01T12:55:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.359, "last_started_at": "2026-07-01T12:53:36+09:00", "pid": 8243, "process": {"exists": false, "pid": 8243}, "restart_count": 31, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.878, "sample_count": 1332, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15170, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:55:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T12:55:43+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T12:55:43+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-01T12:55:43.396069+09:00", "mtime_epoch": 1782878143.396069, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181780}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:55:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.609, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.668, "pushed": 4.668, "read": 4.668}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.188, "count": 140, "max_ms": 14.257, "p95_ms": 10.215}, "gstreamer_push": {"avg_ms": 0.847, "count": 140, "max_ms": 1.007, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.197, "count": 140, "max_ms": 49.138, "p95_ms": 36.448}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.866, "count": 140, "max_ms": 80.793, "p95_ms": 73.972}, "yolo11_640_fire_11class": {"avg_ms": 47.575, "count": 140, "max_ms": 66.894, "p95_ms": 58.208}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.003, "count": 140, "max_ms": 83.5, "p95_ms": 75.045}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.575, "count": 140, "max_ms": 235.549, "p95_ms": 219.359}, "read_interval": {"avg_ms": 214.108, "count": 140, "max_ms": 245.098, "p95_ms": 228.856}, "triton_wait": {"avg_ms": 61.691, "count": 140, "max_ms": 83.513, "p95_ms": 76.127}, "visualize": {"avg_ms": 9.66, "count": 140, "max_ms": 11.168, "p95_ms": 10.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 126}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6091.117, "limit_mb": 8796093022207.996}, "detail": {"pid": 8350, "restart_count": 32, "start_count": 33}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:55:43+09:00", "last_error_summary": "[h264 @ 0x5649dc71c780] error while decoding MB 56 18, bytestream -7", "last_exit_at": "2026-07-01T12:55:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:55:46+09:00", "pid": 8350, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.711, "name": "ffmpeg", "pid": 8350, "rss_mb": 136.711, "state": "S (sleeping)", "threads": 106}, "restart_count": 32, "restart_count_per_hour": 30, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.082, "sample_count": 1310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15170, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:55:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.54, "last_frame_pushed_at": "2026-07-01T12:55:43+09:00", "last_frame_read_age_sec": 4.764, "last_frame_read_at": "2026-07-01T12:55:43+09:00", "mjpeg": {"age_sec": 4.528, "exists": true, "mtime": "2026-07-01T12:55:43.628074+09:00", "mtime_epoch": 1782878143.628074, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181347}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:55:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.609, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.98, "pushed": 3.98, "read": 3.947}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.214, "count": 118, "max_ms": 14.257, "p95_ms": 10.328}, "gstreamer_push": {"avg_ms": 0.845, "count": 119, "max_ms": 1.007, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.278, "count": 118, "max_ms": 49.138, "p95_ms": 39.004}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.149, "count": 118, "max_ms": 80.793, "p95_ms": 73.975}, "yolo11_640_fire_11class": {"avg_ms": 47.813, "count": 118, "max_ms": 66.894, "p95_ms": 61.743}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.324, "count": 118, "max_ms": 83.5, "p95_ms": 76.142}}, "truncated_count": 0}, "processing_total": {"avg_ms": 205.282, "count": 119, "max_ms": 235.549, "p95_ms": 219.691}, "read_interval": {"avg_ms": 214.813, "count": 118, "max_ms": 245.098, "p95_ms": 230.181}, "triton_wait": {"avg_ms": 62.117, "count": 118, "max_ms": 83.513, "p95_ms": 76.783}, "visualize": {"avg_ms": 9.636, "count": 118, "max_ms": 10.994, "p95_ms": 10.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5988.055, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5582a6ed03c0] error while decoding MB 74 41, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:57:53+09:00", "last_error_summary": "[h264 @ 0x5582a6ed03c0] error while decoding MB 74 41, bytestream -9", "last_exit_at": "2026-07-01T12:55:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.385, "last_started_at": "2026-07-01T12:55:46+09:00", "pid": 8350, "process": {"exists": false, "pid": 8350}, "restart_count": 32, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.907, "sample_count": 1321, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15731, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:57:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T12:57:53+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T12:57:53+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T12:57:53.498832+09:00", "mtime_epoch": 1782878273.4988325, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194364}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:57:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.867, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.352, "pushed": 4.352, "read": 4.352}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.299, "count": 129, "max_ms": 15.004, "p95_ms": 13.566}, "gstreamer_push": {"avg_ms": 0.87, "count": 130, "max_ms": 1.24, "p95_ms": 0.954}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.106, "count": 129, "max_ms": 45.271, "p95_ms": 41.441}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.788, "count": 129, "max_ms": 82.829, "p95_ms": 77.247}, "yolo11_640_fire_11class": {"avg_ms": 48.381, "count": 129, "max_ms": 65.455, "p95_ms": 62.715}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.222, "count": 129, "max_ms": 85.229, "p95_ms": 78.734}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.157, "count": 130, "max_ms": 271.48, "p95_ms": 243.324}, "read_interval": {"avg_ms": 231.03, "count": 130, "max_ms": 282.768, "p95_ms": 252.839}, "triton_wait": {"avg_ms": 64.271, "count": 129, "max_ms": 85.241, "p95_ms": 79.14}, "visualize": {"avg_ms": 10.446, "count": 129, "max_ms": 14.383, "p95_ms": 13.673}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5987.227, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T12:57:53+09:00", "last_error_summary": "[h264 @ 0x5582a6ed03c0] error while decoding MB 74 41, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:57:53+09:00", "last_error_summary": "[h264 @ 0x5582a6ed03c0] error while decoding MB 74 41, bytestream -9", "last_exit_at": "2026-07-01T12:57:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.541, "last_started_at": "2026-07-01T12:55:46+09:00", "pid": 8350, "process": {"exists": false, "pid": 8350}, "restart_count": 32, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.907, "sample_count": 1321, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15732, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:57:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T12:57:53+09:00", "last_frame_read_age_sec": 0.207, "last_frame_read_at": "2026-07-01T12:57:53+09:00", "mjpeg": {"age_sec": 0.197, "exists": true, "mtime": "2026-07-01T12:57:53.498832+09:00", "mtime_epoch": 1782878273.4988325, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194364}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:57:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.867, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.364, "pushed": 4.364, "read": 4.331}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.289, "count": 130, "max_ms": 15.004, "p95_ms": 13.566}, "gstreamer_push": {"avg_ms": 0.87, "count": 130, "max_ms": 1.24, "p95_ms": 0.954}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.084, "count": 130, "max_ms": 45.271, "p95_ms": 41.441}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.766, "count": 130, "max_ms": 82.829, "p95_ms": 77.247}, "yolo11_640_fire_11class": {"avg_ms": 48.32, "count": 130, "max_ms": 65.455, "p95_ms": 62.715}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.181, "count": 130, "max_ms": 85.229, "p95_ms": 78.734}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.043, "count": 130, "max_ms": 271.48, "p95_ms": 243.324}, "read_interval": {"avg_ms": 231.064, "count": 129, "max_ms": 282.768, "p95_ms": 252.839}, "triton_wait": {"avg_ms": 64.237, "count": 130, "max_ms": 85.241, "p95_ms": 79.14}, "visualize": {"avg_ms": 10.443, "count": 130, "max_ms": 14.383, "p95_ms": 13.673}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.786, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5987.219, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:57:53+09:00", "last_error_summary": "[h264 @ 0x5582a6ed03c0] error while decoding MB 74 41, bytestream -9", "last_exit_at": "2026-07-01T12:57:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.545, "last_started_at": "2026-07-01T12:55:46+09:00", "pid": 8350, "process": {"exists": false, "pid": 8350}, "restart_count": 32, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.907, "sample_count": 1321, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15732, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:57:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T12:57:53+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T12:57:53+09:00", "mjpeg": {"age_sec": 0.201, "exists": true, "mtime": "2026-07-01T12:57:53.498832+09:00", "mtime_epoch": 1782878273.4988325, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194364}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:57:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.867, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.364, "pushed": 4.364, "read": 4.33}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.289, "count": 130, "max_ms": 15.004, "p95_ms": 13.566}, "gstreamer_push": {"avg_ms": 0.87, "count": 130, "max_ms": 1.24, "p95_ms": 0.954}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.084, "count": 130, "max_ms": 45.271, "p95_ms": 41.441}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.766, "count": 130, "max_ms": 82.829, "p95_ms": 77.247}, "yolo11_640_fire_11class": {"avg_ms": 48.32, "count": 130, "max_ms": 65.455, "p95_ms": 62.715}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.181, "count": 130, "max_ms": 85.229, "p95_ms": 78.734}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.043, "count": 130, "max_ms": 271.48, "p95_ms": 243.324}, "read_interval": {"avg_ms": 231.064, "count": 129, "max_ms": 282.768, "p95_ms": 252.839}, "triton_wait": {"avg_ms": 64.237, "count": 130, "max_ms": 85.241, "p95_ms": 79.14}, "visualize": {"avg_ms": 10.443, "count": 130, "max_ms": 14.383, "p95_ms": 13.673}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.79, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 131}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6095.402, "limit_mb": 8796093022207.996}, "detail": {"pid": 8457, "restart_count": 33, "start_count": 34}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:57:53+09:00", "last_error_summary": "[h264 @ 0x5582a6ed03c0] error while decoding MB 74 41, bytestream -9", "last_exit_at": "2026-07-01T12:57:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T12:57:56+09:00", "pid": 8457, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.18, "name": "ffmpeg", "pid": 8457, "rss_mb": 139.18, "state": "S (sleeping)", "threads": 106}, "restart_count": 33, "restart_count_per_hour": 30, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.947, "sample_count": 1301, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15732, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T12:57:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.454, "last_frame_pushed_at": "2026-07-01T12:57:53+09:00", "last_frame_read_age_sec": 4.658, "last_frame_read_at": "2026-07-01T12:57:53+09:00", "mjpeg": {"age_sec": 4.436, "exists": true, "mtime": "2026-07-01T12:57:53.710837+09:00", "mtime_epoch": 1782878273.710837, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183201}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T12:57:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2357.867, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.743, "pushed": 3.743, "read": 3.71}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.356, "count": 111, "max_ms": 15.004, "p95_ms": 13.671}, "gstreamer_push": {"avg_ms": 0.87, "count": 112, "max_ms": 1.24, "p95_ms": 0.957}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.876, "count": 111, "max_ms": 43.669, "p95_ms": 41.149}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.362, "count": 111, "max_ms": 82.829, "p95_ms": 75.629}, "yolo11_640_fire_11class": {"avg_ms": 48.049, "count": 111, "max_ms": 65.455, "p95_ms": 62.715}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.798, "count": 111, "max_ms": 85.229, "p95_ms": 78.734}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.756, "count": 112, "max_ms": 246.769, "p95_ms": 243.324}, "read_interval": {"avg_ms": 229.802, "count": 111, "max_ms": 257.904, "p95_ms": 252.839}, "triton_wait": {"avg_ms": 63.886, "count": 111, "max_ms": 85.241, "p95_ms": 79.106}, "visualize": {"avg_ms": 10.378, "count": 111, "max_ms": 14.383, "p95_ms": 13.704}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5989.215, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559e4f1b9fc0] error while decoding MB 77 19, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:00:02+09:00", "last_error_summary": "[h264 @ 0x559e4f1b9fc0] error while decoding MB 77 19, bytestream -11", "last_exit_at": "2026-07-01T12:57:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.631, "last_started_at": "2026-07-01T12:57:56+09:00", "pid": 8457, "process": {"exists": false, "pid": 8457}, "restart_count": 33, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.576, "sample_count": 1326, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:00:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T13:00:02+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-01T13:00:02+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T13:00:02.737526+09:00", "mtime_epoch": 1782878402.7375257, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192429}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:00:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.125, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.699, "pushed": 4.699, "read": 4.699}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.385, "count": 140, "max_ms": 12.789, "p95_ms": 10.24}, "gstreamer_push": {"avg_ms": 0.862, "count": 140, "max_ms": 1.223, "p95_ms": 0.895}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.156, "count": 140, "max_ms": 49.182, "p95_ms": 42.716}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.653, "count": 140, "max_ms": 77.171, "p95_ms": 72.339}, "yolo11_640_fire_11class": {"avg_ms": 47.89, "count": 140, "max_ms": 64.363, "p95_ms": 60.528}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.851, "count": 140, "max_ms": 81.0, "p95_ms": 75.389}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.155, "count": 140, "max_ms": 227.59, "p95_ms": 218.506}, "read_interval": {"avg_ms": 214.01, "count": 140, "max_ms": 237.423, "p95_ms": 227.526}, "triton_wait": {"avg_ms": 61.152, "count": 140, "max_ms": 81.013, "p95_ms": 76.162}, "visualize": {"avg_ms": 9.781, "count": 140, "max_ms": 12.399, "p95_ms": 10.487}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.791, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5988.566, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:00:02+09:00", "last_error_summary": "[h264 @ 0x559e4f1b9fc0] error while decoding MB 77 19, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:00:02+09:00", "last_error_summary": "[h264 @ 0x559e4f1b9fc0] error while decoding MB 77 19, bytestream -11", "last_exit_at": "2026-07-01T13:00:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.802, "last_started_at": "2026-07-01T12:57:56+09:00", "pid": 8457, "process": {"exists": false, "pid": 8457}, "restart_count": 33, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.576, "sample_count": 1326, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:00:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:00:02+09:00", "last_frame_read_age_sec": 0.219, "last_frame_read_at": "2026-07-01T13:00:02+09:00", "mjpeg": {"age_sec": 0.21, "exists": true, "mtime": "2026-07-01T13:00:02.737526+09:00", "mtime_epoch": 1782878402.7375257, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192429}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:00:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.125, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.706, "pushed": 4.706, "read": 4.673}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.384, "count": 140, "max_ms": 12.789, "p95_ms": 10.24}, "gstreamer_push": {"avg_ms": 0.862, "count": 141, "max_ms": 1.223, "p95_ms": 0.895}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.155, "count": 140, "max_ms": 49.182, "p95_ms": 42.716}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.589, "count": 140, "max_ms": 77.171, "p95_ms": 71.278}, "yolo11_640_fire_11class": {"avg_ms": 47.939, "count": 140, "max_ms": 64.363, "p95_ms": 60.613}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.862, "count": 140, "max_ms": 81.0, "p95_ms": 75.389}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.23, "count": 141, "max_ms": 227.59, "p95_ms": 218.506}, "read_interval": {"avg_ms": 214.01, "count": 140, "max_ms": 237.423, "p95_ms": 227.526}, "triton_wait": {"avg_ms": 61.143, "count": 140, "max_ms": 81.013, "p95_ms": 76.162}, "visualize": {"avg_ms": 9.78, "count": 140, "max_ms": 12.399, "p95_ms": 10.487}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5988.555, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:00:02+09:00", "last_error_summary": "[h264 @ 0x559e4f1b9fc0] error while decoding MB 77 19, bytestream -11", "last_exit_at": "2026-07-01T13:00:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.805, "last_started_at": "2026-07-01T12:57:56+09:00", "pid": 8457, "process": {"exists": false, "pid": 8457}, "restart_count": 33, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.576, "sample_count": 1326, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:00:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:00:02+09:00", "last_frame_read_age_sec": 0.223, "last_frame_read_at": "2026-07-01T13:00:02+09:00", "mjpeg": {"age_sec": 0.214, "exists": true, "mtime": "2026-07-01T13:00:02.737526+09:00", "mtime_epoch": 1782878402.7375257, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192429}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:00:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.125, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.705, "pushed": 4.705, "read": 4.672}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.384, "count": 140, "max_ms": 12.789, "p95_ms": 10.24}, "gstreamer_push": {"avg_ms": 0.862, "count": 141, "max_ms": 1.223, "p95_ms": 0.895}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.155, "count": 140, "max_ms": 49.182, "p95_ms": 42.716}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.589, "count": 140, "max_ms": 77.171, "p95_ms": 71.278}, "yolo11_640_fire_11class": {"avg_ms": 47.939, "count": 140, "max_ms": 64.363, "p95_ms": 60.613}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.862, "count": 140, "max_ms": 81.0, "p95_ms": 75.389}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.23, "count": 141, "max_ms": 227.59, "p95_ms": 218.506}, "read_interval": {"avg_ms": 214.01, "count": 140, "max_ms": 237.423, "p95_ms": 227.526}, "triton_wait": {"avg_ms": 61.143, "count": 140, "max_ms": 81.013, "p95_ms": 76.162}, "visualize": {"avg_ms": 9.78, "count": 140, "max_ms": 12.399, "p95_ms": 10.487}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 135}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6096.492, "limit_mb": 8796093022207.996}, "detail": {"pid": 8564, "restart_count": 34, "start_count": 35}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:00:02+09:00", "last_error_summary": "[h264 @ 0x559e4f1b9fc0] error while decoding MB 77 19, bytestream -11", "last_exit_at": "2026-07-01T13:00:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:00:05+09:00", "pid": 8564, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.516, "name": "ffmpeg", "pid": 8564, "rss_mb": 138.516, "state": "S (sleeping)", "threads": 106}, "restart_count": 34, "restart_count_per_hour": 30, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.608, "sample_count": 1303, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16305, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:00:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.936, "last_frame_pushed_at": "2026-07-01T13:00:02+09:00", "last_frame_read_age_sec": 5.152, "last_frame_read_at": "2026-07-01T13:00:02+09:00", "mjpeg": {"age_sec": 4.919, "exists": true, "mtime": "2026-07-01T13:00:02.961530+09:00", "mtime_epoch": 1782878402.9615304, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192301}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:00:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.125, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.937, "pushed": 3.937, "read": 3.903}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.348, "count": 117, "max_ms": 12.789, "p95_ms": 10.581}, "gstreamer_push": {"avg_ms": 0.862, "count": 118, "max_ms": 1.223, "p95_ms": 0.919}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.231, "count": 117, "max_ms": 49.182, "p95_ms": 42.822}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.619, "count": 117, "max_ms": 77.171, "p95_ms": 72.339}, "yolo11_640_fire_11class": {"avg_ms": 47.977, "count": 117, "max_ms": 64.363, "p95_ms": 60.832}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.755, "count": 117, "max_ms": 81.0, "p95_ms": 76.149}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.104, "count": 118, "max_ms": 224.656, "p95_ms": 218.506}, "read_interval": {"avg_ms": 213.868, "count": 117, "max_ms": 234.431, "p95_ms": 227.526}, "triton_wait": {"avg_ms": 61.071, "count": 117, "max_ms": 81.013, "p95_ms": 76.782}, "visualize": {"avg_ms": 9.814, "count": 117, "max_ms": 12.399, "p95_ms": 10.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 139}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5990.195, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558742191900] error while decoding MB 50 26, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:01:53+09:00", "last_error_summary": "[h264 @ 0x558742191900] error while decoding MB 50 26, bytestream -11", "last_exit_at": "2026-07-01T13:00:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.802, "last_started_at": "2026-07-01T13:00:05+09:00", "pid": 8564, "process": {"exists": false, "pid": 8564}, "restart_count": 34, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.481, "sample_count": 1310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:01:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T13:01:53+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T13:01:53+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T13:01:53.639804+09:00", "mtime_epoch": 1782878513.6398044, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185138}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:01:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.383, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.612, "pushed": 4.612, "read": 4.612}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.442, "count": 137, "max_ms": 11.741, "p95_ms": 10.286}, "gstreamer_push": {"avg_ms": 0.864, "count": 138, "max_ms": 1.281, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.882, "count": 137, "max_ms": 49.61, "p95_ms": 41.105}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.134, "count": 137, "max_ms": 81.605, "p95_ms": 75.701}, "yolo11_640_fire_11class": {"avg_ms": 46.335, "count": 137, "max_ms": 61.313, "p95_ms": 57.822}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.133, "count": 137, "max_ms": 85.088, "p95_ms": 75.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.181, "count": 138, "max_ms": 234.893, "p95_ms": 225.094}, "read_interval": {"avg_ms": 217.958, "count": 138, "max_ms": 244.314, "p95_ms": 234.648}, "triton_wait": {"avg_ms": 64.207, "count": 137, "max_ms": 85.1, "p95_ms": 78.167}, "visualize": {"avg_ms": 9.796, "count": 137, "max_ms": 11.167, "p95_ms": 10.888}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 139}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5989.637, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:01:53+09:00", "last_error_summary": "[h264 @ 0x558742191900] error while decoding MB 50 26, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:01:53+09:00", "last_error_summary": "[h264 @ 0x558742191900] error while decoding MB 50 26, bytestream -11", "last_exit_at": "2026-07-01T13:01:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.959, "last_started_at": "2026-07-01T13:00:05+09:00", "pid": 8564, "process": {"exists": false, "pid": 8564}, "restart_count": 34, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.481, "sample_count": 1310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16779, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:01:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:01:53+09:00", "last_frame_read_age_sec": 0.209, "last_frame_read_at": "2026-07-01T13:01:53+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T13:01:53.639804+09:00", "mtime_epoch": 1782878513.6398044, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185138}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:01:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.383, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.621, "pushed": 4.621, "read": 4.587}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.466, "count": 138, "max_ms": 12.742, "p95_ms": 10.347}, "gstreamer_push": {"avg_ms": 0.864, "count": 138, "max_ms": 1.281, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.883, "count": 138, "max_ms": 49.61, "p95_ms": 41.105}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.054, "count": 138, "max_ms": 81.605, "p95_ms": 75.701}, "yolo11_640_fire_11class": {"avg_ms": 46.306, "count": 138, "max_ms": 61.313, "p95_ms": 57.822}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.085, "count": 138, "max_ms": 85.088, "p95_ms": 75.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.163, "count": 138, "max_ms": 234.893, "p95_ms": 225.094}, "read_interval": {"avg_ms": 217.97, "count": 137, "max_ms": 244.314, "p95_ms": 234.648}, "triton_wait": {"avg_ms": 64.151, "count": 138, "max_ms": 85.1, "p95_ms": 78.167}, "visualize": {"avg_ms": 9.796, "count": 138, "max_ms": 11.167, "p95_ms": 10.888}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 139}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5989.621, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:01:53+09:00", "last_error_summary": "[h264 @ 0x558742191900] error while decoding MB 50 26, bytestream -11", "last_exit_at": "2026-07-01T13:01:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.963, "last_started_at": "2026-07-01T13:00:05+09:00", "pid": 8564, "process": {"exists": false, "pid": 8564}, "restart_count": 34, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.481, "sample_count": 1310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16779, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:01:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T13:01:53+09:00", "last_frame_read_age_sec": 0.212, "last_frame_read_at": "2026-07-01T13:01:53+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-01T13:01:53.639804+09:00", "mtime_epoch": 1782878513.6398044, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185138}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:01:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.383, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.62, "pushed": 4.62, "read": 4.587}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.466, "count": 138, "max_ms": 12.742, "p95_ms": 10.347}, "gstreamer_push": {"avg_ms": 0.864, "count": 138, "max_ms": 1.281, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.883, "count": 138, "max_ms": 49.61, "p95_ms": 41.105}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.054, "count": 138, "max_ms": 81.605, "p95_ms": 75.701}, "yolo11_640_fire_11class": {"avg_ms": 46.306, "count": 138, "max_ms": 61.313, "p95_ms": 57.822}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.085, "count": 138, "max_ms": 85.088, "p95_ms": 75.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.163, "count": 138, "max_ms": 234.893, "p95_ms": 225.094}, "read_interval": {"avg_ms": 217.97, "count": 137, "max_ms": 244.314, "p95_ms": 234.648}, "triton_wait": {"avg_ms": 64.151, "count": 138, "max_ms": 85.1, "p95_ms": 78.167}, "visualize": {"avg_ms": 9.796, "count": 138, "max_ms": 11.167, "p95_ms": 10.888}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 139}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6097.727, "limit_mb": 8796093022207.996}, "detail": {"pid": 8671, "restart_count": 35, "start_count": 36}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:01:53+09:00", "last_error_summary": "[h264 @ 0x558742191900] error while decoding MB 50 26, bytestream -11", "last_exit_at": "2026-07-01T13:01:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:01:56+09:00", "pid": 8671, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.844, "name": "ffmpeg", "pid": 8671, "rss_mb": 138.844, "state": "S (sleeping)", "threads": 106}, "restart_count": 35, "restart_count_per_hour": 30, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.481, "sample_count": 1290, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16779, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:01:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.569, "last_frame_pushed_at": "2026-07-01T13:01:53+09:00", "last_frame_read_age_sec": 4.775, "last_frame_read_at": "2026-07-01T13:01:53+09:00", "mjpeg": {"age_sec": 4.55, "exists": true, "mtime": "2026-07-01T13:01:53.855809+09:00", "mtime_epoch": 1782878513.8558087, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185352}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:01:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.383, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.921, "pushed": 3.921, "read": 3.888}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.453, "count": 117, "max_ms": 12.742, "p95_ms": 10.347}, "gstreamer_push": {"avg_ms": 0.862, "count": 117, "max_ms": 1.264, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.882, "count": 117, "max_ms": 49.61, "p95_ms": 39.704}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.841, "count": 117, "max_ms": 81.605, "p95_ms": 75.701}, "yolo11_640_fire_11class": {"avg_ms": 46.022, "count": 117, "max_ms": 61.313, "p95_ms": 58.01}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.82, "count": 117, "max_ms": 85.088, "p95_ms": 75.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 207.981, "count": 117, "max_ms": 234.893, "p95_ms": 225.839}, "read_interval": {"avg_ms": 217.805, "count": 116, "max_ms": 244.314, "p95_ms": 235.251}, "triton_wait": {"avg_ms": 63.804, "count": 117, "max_ms": 85.1, "p95_ms": 77.813}, "visualize": {"avg_ms": 9.801, "count": 117, "max_ms": 11.167, "p95_ms": 10.888}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.837, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5991.305, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5643260a7680] error while decoding MB 67 14, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:03:43+09:00", "last_error_summary": "[h264 @ 0x5643260a7680] error while decoding MB 67 14, bytestream -5", "last_exit_at": "2026-07-01T13:01:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.11, "last_started_at": "2026-07-01T13:01:56+09:00", "pid": 8671, "process": {"exists": false, "pid": 8671}, "restart_count": 35, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.784, "sample_count": 1302, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17242, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T13:03:43+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T13:03:43+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T13:03:43.474038+09:00", "mtime_epoch": 1782878623.4740376, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185929}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:03:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.641, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.61, "pushed": 4.61, "read": 4.61}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.65, "count": 137, "max_ms": 12.174, "p95_ms": 10.802}, "gstreamer_push": {"avg_ms": 0.85, "count": 138, "max_ms": 1.025, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.931, "count": 137, "max_ms": 51.34, "p95_ms": 42.756}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.716, "count": 137, "max_ms": 74.292, "p95_ms": 70.802}, "yolo11_640_fire_11class": {"avg_ms": 46.57, "count": 137, "max_ms": 63.567, "p95_ms": 58.653}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.594, "count": 137, "max_ms": 77.926, "p95_ms": 73.658}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.603, "count": 138, "max_ms": 234.476, "p95_ms": 225.019}, "read_interval": {"avg_ms": 218.085, "count": 138, "max_ms": 245.296, "p95_ms": 234.886}, "triton_wait": {"avg_ms": 60.956, "count": 137, "max_ms": 77.938, "p95_ms": 73.826}, "visualize": {"avg_ms": 9.95, "count": 137, "max_ms": 12.871, "p95_ms": 11.007}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5990.656, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:03:43+09:00", "last_error_summary": "[h264 @ 0x5643260a7680] error while decoding MB 67 14, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:03:43+09:00", "last_error_summary": "[h264 @ 0x5643260a7680] error while decoding MB 67 14, bytestream -5", "last_exit_at": "2026-07-01T13:03:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.267, "last_started_at": "2026-07-01T13:01:56+09:00", "pid": 8671, "process": {"exists": false, "pid": 8671}, "restart_count": 35, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.784, "sample_count": 1303, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T13:03:43+09:00", "last_frame_read_age_sec": 0.209, "last_frame_read_at": "2026-07-01T13:03:43+09:00", "mjpeg": {"age_sec": 0.197, "exists": true, "mtime": "2026-07-01T13:03:43.474038+09:00", "mtime_epoch": 1782878623.4740376, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185929}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:03:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.641, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.622, "pushed": 4.622, "read": 4.588}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.647, "count": 138, "max_ms": 12.174, "p95_ms": 10.802}, "gstreamer_push": {"avg_ms": 0.85, "count": 138, "max_ms": 1.025, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.935, "count": 138, "max_ms": 51.34, "p95_ms": 42.756}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.733, "count": 138, "max_ms": 74.292, "p95_ms": 70.802}, "yolo11_640_fire_11class": {"avg_ms": 46.597, "count": 138, "max_ms": 63.567, "p95_ms": 58.653}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.603, "count": 138, "max_ms": 77.926, "p95_ms": 73.658}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.519, "count": 138, "max_ms": 234.476, "p95_ms": 225.019}, "read_interval": {"avg_ms": 218.046, "count": 137, "max_ms": 245.296, "p95_ms": 234.886}, "triton_wait": {"avg_ms": 60.963, "count": 138, "max_ms": 77.938, "p95_ms": 73.826}, "visualize": {"avg_ms": 9.949, "count": 138, "max_ms": 12.871, "p95_ms": 11.007}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.86, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5990.648, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:03:43+09:00", "last_error_summary": "[h264 @ 0x5643260a7680] error while decoding MB 67 14, bytestream -5", "last_exit_at": "2026-07-01T13:03:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.27, "last_started_at": "2026-07-01T13:01:56+09:00", "pid": 8671, "process": {"exists": false, "pid": 8671}, "restart_count": 35, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.784, "sample_count": 1303, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T13:03:43+09:00", "last_frame_read_age_sec": 0.212, "last_frame_read_at": "2026-07-01T13:03:43+09:00", "mjpeg": {"age_sec": 0.201, "exists": true, "mtime": "2026-07-01T13:03:43.474038+09:00", "mtime_epoch": 1782878623.4740376, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185929}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:03:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.641, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.621, "pushed": 4.621, "read": 4.588}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.642, "count": 137, "max_ms": 12.174, "p95_ms": 10.802}, "gstreamer_push": {"avg_ms": 0.85, "count": 138, "max_ms": 1.025, "p95_ms": 0.901}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.941, "count": 137, "max_ms": 51.34, "p95_ms": 42.756}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.715, "count": 137, "max_ms": 74.292, "p95_ms": 70.802}, "yolo11_640_fire_11class": {"avg_ms": 46.612, "count": 137, "max_ms": 63.567, "p95_ms": 58.653}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.611, "count": 137, "max_ms": 77.926, "p95_ms": 73.658}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.519, "count": 138, "max_ms": 234.476, "p95_ms": 225.019}, "read_interval": {"avg_ms": 218.046, "count": 137, "max_ms": 245.296, "p95_ms": 234.886}, "triton_wait": {"avg_ms": 60.968, "count": 137, "max_ms": 77.938, "p95_ms": 73.826}, "visualize": {"avg_ms": 9.943, "count": 137, "max_ms": 12.871, "p95_ms": 11.007}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.863, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 143}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6104.406, "limit_mb": 8796093022207.996}, "detail": {"pid": 8778, "restart_count": 36, "start_count": 37}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:03:43+09:00", "last_error_summary": "[h264 @ 0x5643260a7680] error while decoding MB 67 14, bytestream -5", "last_exit_at": "2026-07-01T13:03:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:03:46+09:00", "pid": 8778, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.445, "name": "ffmpeg", "pid": 8778, "rss_mb": 144.445, "state": "S (sleeping)", "threads": 106}, "restart_count": 36, "restart_count_per_hour": 30, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.941, "sample_count": 1279, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.984, "last_frame_pushed_at": "2026-07-01T13:03:43+09:00", "last_frame_read_age_sec": 5.188, "last_frame_read_at": "2026-07-01T13:03:43+09:00", "mjpeg": {"age_sec": 4.969, "exists": true, "mtime": "2026-07-01T13:03:43.682042+09:00", "mtime_epoch": 1782878623.682042, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179642}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:03:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.641, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.887, "pushed": 3.887, "read": 3.854}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.552, "count": 116, "max_ms": 12.174, "p95_ms": 10.623}, "gstreamer_push": {"avg_ms": 0.847, "count": 116, "max_ms": 0.959, "p95_ms": 0.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.925, "count": 116, "max_ms": 51.34, "p95_ms": 42.756}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.389, "count": 116, "max_ms": 72.756, "p95_ms": 70.802}, "yolo11_640_fire_11class": {"avg_ms": 46.751, "count": 116, "max_ms": 63.567, "p95_ms": 58.653}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.591, "count": 116, "max_ms": 77.926, "p95_ms": 73.611}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.951, "count": 116, "max_ms": 233.016, "p95_ms": 225.019}, "read_interval": {"avg_ms": 216.411, "count": 115, "max_ms": 243.784, "p95_ms": 234.758}, "triton_wait": {"avg_ms": 60.855, "count": 116, "max_ms": 77.938, "p95_ms": 73.623}, "visualize": {"avg_ms": 9.876, "count": 116, "max_ms": 12.871, "p95_ms": 10.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.842, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 147}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5992.027, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561a9d170240] error while decoding MB 31 25, bytestream -22", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:05:34+09:00", "last_error_summary": "[h264 @ 0x561a9d170240] error while decoding MB 31 25, bytestream -22", "last_exit_at": "2026-07-01T13:03:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.12, "last_started_at": "2026-07-01T13:03:46+09:00", "pid": 8778, "process": {"exists": false, "pid": 8778}, "restart_count": 36, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.402, "sample_count": 1287, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17711, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:05:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T13:05:34+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T13:05:34+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T13:05:34.724281+09:00", "mtime_epoch": 1782878734.7242808, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185412}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:05:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.898, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.518, "pushed": 4.518, "read": 4.518}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.741, "count": 134, "max_ms": 12.729, "p95_ms": 10.423}, "gstreamer_push": {"avg_ms": 0.84, "count": 135, "max_ms": 0.909, "p95_ms": 0.879}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.125, "count": 134, "max_ms": 49.152, "p95_ms": 43.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.493, "count": 134, "max_ms": 74.785, "p95_ms": 73.369}, "yolo11_640_fire_11class": {"avg_ms": 47.573, "count": 134, "max_ms": 64.729, "p95_ms": 60.141}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.193, "count": 134, "max_ms": 78.422, "p95_ms": 74.549}}, "truncated_count": 0}, "processing_total": {"avg_ms": 212.349, "count": 135, "max_ms": 244.118, "p95_ms": 235.347}, "read_interval": {"avg_ms": 222.58, "count": 135, "max_ms": 254.829, "p95_ms": 245.235}, "triton_wait": {"avg_ms": 61.548, "count": 134, "max_ms": 78.433, "p95_ms": 74.698}, "visualize": {"avg_ms": 10.022, "count": 134, "max_ms": 11.371, "p95_ms": 11.018}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 147}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5991.555, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:05:34+09:00", "last_error_summary": "[h264 @ 0x561a9d170240] error while decoding MB 31 25, bytestream -22", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:05:34+09:00", "last_error_summary": "[h264 @ 0x561a9d170240] error while decoding MB 31 25, bytestream -22", "last_exit_at": "2026-07-01T13:05:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.303, "last_started_at": "2026-07-01T13:03:46+09:00", "pid": 8778, "process": {"exists": false, "pid": 8778}, "restart_count": 36, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.402, "sample_count": 1288, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17712, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:05:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:05:34+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T13:05:34+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T13:05:34.724281+09:00", "mtime_epoch": 1782878734.7242808, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185412}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:05:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.898, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.525, "pushed": 4.525, "read": 4.492}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.764, "count": 135, "max_ms": 12.85, "p95_ms": 10.457}, "gstreamer_push": {"avg_ms": 0.84, "count": 135, "max_ms": 0.909, "p95_ms": 0.878}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.083, "count": 135, "max_ms": 49.152, "p95_ms": 43.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.465, "count": 135, "max_ms": 74.785, "p95_ms": 73.369}, "yolo11_640_fire_11class": {"avg_ms": 47.552, "count": 135, "max_ms": 64.729, "p95_ms": 60.141}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.167, "count": 135, "max_ms": 78.422, "p95_ms": 74.549}}, "truncated_count": 0}, "processing_total": {"avg_ms": 212.446, "count": 135, "max_ms": 244.118, "p95_ms": 235.347}, "read_interval": {"avg_ms": 222.539, "count": 134, "max_ms": 254.829, "p95_ms": 245.235}, "triton_wait": {"avg_ms": 61.519, "count": 135, "max_ms": 78.433, "p95_ms": 74.698}, "visualize": {"avg_ms": 10.038, "count": 135, "max_ms": 12.135, "p95_ms": 11.051}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.833, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 147}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5991.547, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:05:34+09:00", "last_error_summary": "[h264 @ 0x561a9d170240] error while decoding MB 31 25, bytestream -22", "last_exit_at": "2026-07-01T13:05:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.306, "last_started_at": "2026-07-01T13:03:46+09:00", "pid": 8778, "process": {"exists": false, "pid": 8778}, "restart_count": 36, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.402, "sample_count": 1288, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17712, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:05:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T13:05:34+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T13:05:34+09:00", "mjpeg": {"age_sec": 0.232, "exists": true, "mtime": "2026-07-01T13:05:34.724281+09:00", "mtime_epoch": 1782878734.7242808, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185412}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:05:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.898, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.525, "pushed": 4.525, "read": 4.491}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.764, "count": 135, "max_ms": 12.85, "p95_ms": 10.457}, "gstreamer_push": {"avg_ms": 0.84, "count": 135, "max_ms": 0.909, "p95_ms": 0.878}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.083, "count": 135, "max_ms": 49.152, "p95_ms": 43.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.465, "count": 135, "max_ms": 74.785, "p95_ms": 73.369}, "yolo11_640_fire_11class": {"avg_ms": 47.552, "count": 135, "max_ms": 64.729, "p95_ms": 60.141}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.167, "count": 135, "max_ms": 78.422, "p95_ms": 74.549}}, "truncated_count": 0}, "processing_total": {"avg_ms": 212.446, "count": 135, "max_ms": 244.118, "p95_ms": 235.347}, "read_interval": {"avg_ms": 222.539, "count": 134, "max_ms": 254.829, "p95_ms": 245.235}, "triton_wait": {"avg_ms": 61.519, "count": 135, "max_ms": 78.433, "p95_ms": 74.698}, "visualize": {"avg_ms": 10.038, "count": 135, "max_ms": 12.135, "p95_ms": 11.051}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.836, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 147}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6099.594, "limit_mb": 8796093022207.996}, "detail": {"pid": 8885, "restart_count": 37, "start_count": 38}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:05:34+09:00", "last_error_summary": "[h264 @ 0x561a9d170240] error while decoding MB 31 25, bytestream -22", "last_exit_at": "2026-07-01T13:05:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:05:37+09:00", "pid": 8885, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.148, "name": "ffmpeg", "pid": 8885, "rss_mb": 139.148, "state": "S (sleeping)", "threads": 106}, "restart_count": 37, "restart_count_per_hour": 30, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.402, "sample_count": 1270, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17712, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:05:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.061, "last_frame_pushed_at": "2026-07-01T13:05:34+09:00", "last_frame_read_age_sec": 4.293, "last_frame_read_at": "2026-07-01T13:05:34+09:00", "mjpeg": {"age_sec": 4.046, "exists": true, "mtime": "2026-07-01T13:05:34.964286+09:00", "mtime_epoch": 1782878734.9642856, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181067}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:05:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2358.898, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.902, "pushed": 3.902, "read": 3.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.772, "count": 116, "max_ms": 12.85, "p95_ms": 10.457}, "gstreamer_push": {"avg_ms": 0.841, "count": 117, "max_ms": 0.909, "p95_ms": 0.884}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.273, "count": 116, "max_ms": 49.152, "p95_ms": 43.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.28, "count": 116, "max_ms": 74.785, "p95_ms": 73.369}, "yolo11_640_fire_11class": {"avg_ms": 47.782, "count": 116, "max_ms": 64.729, "p95_ms": 60.268}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.228, "count": 116, "max_ms": 78.422, "p95_ms": 74.686}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.199, "count": 117, "max_ms": 244.118, "p95_ms": 235.431}, "read_interval": {"avg_ms": 223.169, "count": 116, "max_ms": 254.829, "p95_ms": 245.612}, "triton_wait": {"avg_ms": 61.548, "count": 116, "max_ms": 78.433, "p95_ms": 74.797}, "visualize": {"avg_ms": 10.09, "count": 116, "max_ms": 12.135, "p95_ms": 11.127}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 152}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5992.781, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55637000b080] error while decoding MB 42 10, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:07:45+09:00", "last_error_summary": "[h264 @ 0x55637000b080] error while decoding MB 42 10, bytestream -7", "last_exit_at": "2026-07-01T13:05:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.437, "last_started_at": "2026-07-01T13:05:37+09:00", "pid": 8885, "process": {"exists": false, "pid": 8885}, "restart_count": 37, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.092, "sample_count": 1309, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:07:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T13:07:45+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T13:07:45+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T13:07:45.406896+09:00", "mtime_epoch": 1782878865.4068959, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198344}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:07:45+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.156, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.511, "pushed": 4.511, "read": 4.511}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.719, "count": 134, "max_ms": 12.351, "p95_ms": 10.531}, "gstreamer_push": {"avg_ms": 0.842, "count": 135, "max_ms": 1.139, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.806, "count": 134, "max_ms": 47.154, "p95_ms": 41.963}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.698, "count": 134, "max_ms": 78.32, "p95_ms": 74.469}, "yolo11_640_fire_11class": {"avg_ms": 48.126, "count": 134, "max_ms": 63.612, "p95_ms": 61.3}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.864, "count": 134, "max_ms": 80.823, "p95_ms": 77.263}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.258, "count": 135, "max_ms": 243.702, "p95_ms": 238.032}, "read_interval": {"avg_ms": 222.745, "count": 135, "max_ms": 254.802, "p95_ms": 246.345}, "triton_wait": {"avg_ms": 62.331, "count": 134, "max_ms": 80.835, "p95_ms": 77.397}, "visualize": {"avg_ms": 10.126, "count": 134, "max_ms": 14.037, "p95_ms": 10.914}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 152}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5992.152, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:07:45+09:00", "last_error_summary": "[h264 @ 0x55637000b080] error while decoding MB 42 10, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:07:45+09:00", "last_error_summary": "[h264 @ 0x55637000b080] error while decoding MB 42 10, bytestream -7", "last_exit_at": "2026-07-01T13:07:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.596, "last_started_at": "2026-07-01T13:05:37+09:00", "pid": 8885, "process": {"exists": false, "pid": 8885}, "restart_count": 37, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.092, "sample_count": 1309, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18293, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:07:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T13:07:45+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T13:07:45+09:00", "mjpeg": {"age_sec": 0.198, "exists": true, "mtime": "2026-07-01T13:07:45.406896+09:00", "mtime_epoch": 1782878865.4068959, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198344}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:07:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.156, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.52, "pushed": 4.52, "read": 4.486}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.715, "count": 135, "max_ms": 12.351, "p95_ms": 10.531}, "gstreamer_push": {"avg_ms": 0.842, "count": 135, "max_ms": 1.139, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.785, "count": 135, "max_ms": 47.154, "p95_ms": 41.963}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.713, "count": 135, "max_ms": 78.32, "p95_ms": 74.469}, "yolo11_640_fire_11class": {"avg_ms": 48.132, "count": 135, "max_ms": 63.612, "p95_ms": 61.3}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.863, "count": 135, "max_ms": 80.823, "p95_ms": 77.263}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.316, "count": 135, "max_ms": 243.702, "p95_ms": 238.032}, "read_interval": {"avg_ms": 222.856, "count": 134, "max_ms": 254.802, "p95_ms": 246.345}, "triton_wait": {"avg_ms": 62.327, "count": 135, "max_ms": 80.835, "p95_ms": 77.397}, "visualize": {"avg_ms": 10.123, "count": 135, "max_ms": 14.037, "p95_ms": 10.914}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 152}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5991.996, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:07:45+09:00", "last_error_summary": "[h264 @ 0x55637000b080] error while decoding MB 42 10, bytestream -7", "last_exit_at": "2026-07-01T13:07:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.599, "last_started_at": "2026-07-01T13:05:37+09:00", "pid": 8885, "process": {"exists": false, "pid": 8885}, "restart_count": 37, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.092, "sample_count": 1309, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18293, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:07:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:07:45+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T13:07:45+09:00", "mjpeg": {"age_sec": 0.202, "exists": true, "mtime": "2026-07-01T13:07:45.406896+09:00", "mtime_epoch": 1782878865.4068959, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198344}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:07:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.156, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.519, "pushed": 4.519, "read": 4.486}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.715, "count": 135, "max_ms": 12.351, "p95_ms": 10.531}, "gstreamer_push": {"avg_ms": 0.842, "count": 135, "max_ms": 1.139, "p95_ms": 0.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.785, "count": 135, "max_ms": 47.154, "p95_ms": 41.963}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.713, "count": 135, "max_ms": 78.32, "p95_ms": 74.469}, "yolo11_640_fire_11class": {"avg_ms": 48.132, "count": 135, "max_ms": 63.612, "p95_ms": 61.3}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.863, "count": 135, "max_ms": 80.823, "p95_ms": 77.263}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.316, "count": 135, "max_ms": 243.702, "p95_ms": 238.032}, "read_interval": {"avg_ms": 222.856, "count": 134, "max_ms": 254.802, "p95_ms": 246.345}, "triton_wait": {"avg_ms": 62.327, "count": 135, "max_ms": 80.835, "p95_ms": 77.397}, "visualize": {"avg_ms": 10.123, "count": 135, "max_ms": 14.037, "p95_ms": 10.914}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 152}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6097.977, "limit_mb": 8796093022207.996}, "detail": {"pid": 8992, "restart_count": 38, "start_count": 39}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:07:45+09:00", "last_error_summary": "[h264 @ 0x55637000b080] error while decoding MB 42 10, bytestream -7", "last_exit_at": "2026-07-01T13:07:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:07:47+09:00", "pid": 8992, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.762, "name": "ffmpeg", "pid": 8992, "rss_mb": 136.762, "state": "S (sleeping)", "threads": 106}, "restart_count": 38, "restart_count_per_hour": 30, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.149, "sample_count": 1290, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18293, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:07:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.219, "last_frame_pushed_at": "2026-07-01T13:07:45+09:00", "last_frame_read_age_sec": 4.426, "last_frame_read_at": "2026-07-01T13:07:45+09:00", "mjpeg": {"age_sec": 4.202, "exists": true, "mtime": "2026-07-01T13:07:45.618900+09:00", "mtime_epoch": 1782878865.6189, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198438}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:07:49+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.156, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.88, "pushed": 3.88, "read": 3.847}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.755, "count": 115, "max_ms": 12.351, "p95_ms": 10.549}, "gstreamer_push": {"avg_ms": 0.843, "count": 116, "max_ms": 1.139, "p95_ms": 0.89}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.798, "count": 115, "max_ms": 47.154, "p95_ms": 40.212}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.627, "count": 115, "max_ms": 78.32, "p95_ms": 75.176}, "yolo11_640_fire_11class": {"avg_ms": 48.492, "count": 115, "max_ms": 63.612, "p95_ms": 61.421}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.781, "count": 115, "max_ms": 80.823, "p95_ms": 77.263}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.661, "count": 116, "max_ms": 243.702, "p95_ms": 238.169}, "read_interval": {"avg_ms": 223.204, "count": 115, "max_ms": 254.802, "p95_ms": 247.218}, "triton_wait": {"avg_ms": 62.174, "count": 115, "max_ms": 80.835, "p95_ms": 78.251}, "visualize": {"avg_ms": 10.175, "count": 115, "max_ms": 14.037, "p95_ms": 10.945}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 157}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5993.887, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5577aaa541c0] error while decoding MB 40 40, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:09:58+09:00", "last_error_summary": "[h264 @ 0x5577aaa541c0] error while decoding MB 40 40, bytestream -5", "last_exit_at": "2026-07-01T13:07:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.402, "last_started_at": "2026-07-01T13:07:47+09:00", "pid": 8992, "process": {"exists": false, "pid": 8992}, "restart_count": 38, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.191, "sample_count": 1331, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18879, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:09:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T13:09:58+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T13:09:58+09:00", "mjpeg": {"age_sec": 0.036, "exists": true, "mtime": "2026-07-01T13:09:58.185536+09:00", "mtime_epoch": 1782878998.185536, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195983}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:09:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.414, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.319, "pushed": 4.319, "read": 4.319}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.24, "count": 128, "max_ms": 15.167, "p95_ms": 11.392}, "gstreamer_push": {"avg_ms": 0.853, "count": 129, "max_ms": 1.328, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.427, "count": 128, "max_ms": 55.492, "p95_ms": 44.797}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.608, "count": 128, "max_ms": 77.218, "p95_ms": 74.092}, "yolo11_640_fire_11class": {"avg_ms": 47.73, "count": 128, "max_ms": 65.064, "p95_ms": 60.596}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.715, "count": 128, "max_ms": 79.543, "p95_ms": 75.858}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.072, "count": 129, "max_ms": 242.56, "p95_ms": 239.088}, "read_interval": {"avg_ms": 232.84, "count": 129, "max_ms": 252.491, "p95_ms": 249.721}, "triton_wait": {"avg_ms": 62.056, "count": 128, "max_ms": 79.558, "p95_ms": 75.919}, "visualize": {"avg_ms": 10.579, "count": 128, "max_ms": 12.613, "p95_ms": 11.481}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 157}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5993.285, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:09:58+09:00", "last_error_summary": "[h264 @ 0x5577aaa541c0] error while decoding MB 40 40, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:09:58+09:00", "last_error_summary": "[h264 @ 0x5577aaa541c0] error while decoding MB 40 40, bytestream -5", "last_exit_at": "2026-07-01T13:09:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.562, "last_started_at": "2026-07-01T13:07:47+09:00", "pid": 8992, "process": {"exists": false, "pid": 8992}, "restart_count": 38, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.176, "sample_count": 1332, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18880, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:09:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:09:58+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T13:09:58+09:00", "mjpeg": {"age_sec": 0.196, "exists": true, "mtime": "2026-07-01T13:09:58.185536+09:00", "mtime_epoch": 1782878998.185536, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195983}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:09:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.414, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.325, "pushed": 4.325, "read": 4.291}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.231, "count": 129, "max_ms": 15.167, "p95_ms": 11.392}, "gstreamer_push": {"avg_ms": 0.853, "count": 129, "max_ms": 1.328, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.476, "count": 129, "max_ms": 55.492, "p95_ms": 44.797}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.626, "count": 129, "max_ms": 77.218, "p95_ms": 74.092}, "yolo11_640_fire_11class": {"avg_ms": 47.706, "count": 129, "max_ms": 65.064, "p95_ms": 60.596}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.721, "count": 129, "max_ms": 79.543, "p95_ms": 75.858}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.98, "count": 129, "max_ms": 242.56, "p95_ms": 239.088}, "read_interval": {"avg_ms": 232.889, "count": 128, "max_ms": 252.491, "p95_ms": 249.721}, "triton_wait": {"avg_ms": 62.059, "count": 129, "max_ms": 79.558, "p95_ms": 75.919}, "visualize": {"avg_ms": 10.574, "count": 129, "max_ms": 12.613, "p95_ms": 11.481}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.826, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 157}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5993.242, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:09:58+09:00", "last_error_summary": "[h264 @ 0x5577aaa541c0] error while decoding MB 40 40, bytestream -5", "last_exit_at": "2026-07-01T13:09:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.565, "last_started_at": "2026-07-01T13:07:47+09:00", "pid": 8992, "process": {"exists": false, "pid": 8992}, "restart_count": 38, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.176, "sample_count": 1332, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18880, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:09:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:09:58+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T13:09:58+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T13:09:58.185536+09:00", "mtime_epoch": 1782878998.185536, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195983}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:09:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.414, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.325, "pushed": 4.325, "read": 4.291}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.231, "count": 129, "max_ms": 15.167, "p95_ms": 11.392}, "gstreamer_push": {"avg_ms": 0.853, "count": 129, "max_ms": 1.328, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.476, "count": 129, "max_ms": 55.492, "p95_ms": 44.797}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.626, "count": 129, "max_ms": 77.218, "p95_ms": 74.092}, "yolo11_640_fire_11class": {"avg_ms": 47.706, "count": 129, "max_ms": 65.064, "p95_ms": 60.596}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.721, "count": 129, "max_ms": 79.543, "p95_ms": 75.858}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.98, "count": 129, "max_ms": 242.56, "p95_ms": 239.088}, "read_interval": {"avg_ms": 232.889, "count": 128, "max_ms": 252.491, "p95_ms": 249.721}, "triton_wait": {"avg_ms": 62.059, "count": 129, "max_ms": 79.558, "p95_ms": 75.919}, "visualize": {"avg_ms": 10.574, "count": 129, "max_ms": 12.613, "p95_ms": 11.481}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.83, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 157}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6098.863, "limit_mb": 8796093022207.996}, "detail": {"pid": 9099, "restart_count": 39, "start_count": 40}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:09:58+09:00", "last_error_summary": "[h264 @ 0x5577aaa541c0] error while decoding MB 40 40, bytestream -5", "last_exit_at": "2026-07-01T13:10:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:10:01+09:00", "pid": 9099, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.746, "name": "ffmpeg", "pid": 9099, "rss_mb": 136.746, "state": "S (sleeping)", "threads": 106}, "restart_count": 39, "restart_count_per_hour": 30, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.176, "sample_count": 1311, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18880, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:09:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.853, "last_frame_pushed_at": "2026-07-01T13:09:58+09:00", "last_frame_read_age_sec": 5.06, "last_frame_read_at": "2026-07-01T13:09:58+09:00", "mjpeg": {"age_sec": 4.834, "exists": true, "mtime": "2026-07-01T13:09:58.397540+09:00", "mtime_epoch": 1782878998.39754, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196041}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:10:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.414, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.619, "pushed": 3.619, "read": 3.586}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.207, "count": 108, "max_ms": 14.373, "p95_ms": 11.381}, "gstreamer_push": {"avg_ms": 0.856, "count": 108, "max_ms": 1.328, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.347, "count": 108, "max_ms": 55.492, "p95_ms": 45.508}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.975, "count": 108, "max_ms": 75.607, "p95_ms": 74.092}, "yolo11_640_fire_11class": {"avg_ms": 47.992, "count": 108, "max_ms": 65.064, "p95_ms": 60.596}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.781, "count": 108, "max_ms": 79.543, "p95_ms": 75.706}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.556, "count": 108, "max_ms": 242.56, "p95_ms": 240.005}, "read_interval": {"avg_ms": 233.512, "count": 107, "max_ms": 252.491, "p95_ms": 249.732}, "triton_wait": {"avg_ms": 62.145, "count": 108, "max_ms": 79.558, "p95_ms": 75.718}, "visualize": {"avg_ms": 10.583, "count": 108, "max_ms": 12.613, "p95_ms": 11.481}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.84, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5938.086, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5648d02e3740] left block unavailable for requested intra mode [h264 @ 0x5648d02e3740] error while decoding MB 0 3, bytestream 35811 [h264 @ 0x5648d05502c0] error while decoding MB 25 9, bytestream -5 [in#0/rtsp @ 0x5648cf92fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cf92fe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:12:27+09:00", "last_error_summary": "[h264 @ 0x5648d02e3740] left block unavailable for requested intra mode [h264 @ 0x5648d02e3740] error while decoding MB 0 3, bytestream 35811 [h264 @ 0x5648d05502c0] error while decoding MB 25 9, bytestream -5 [in#0/rtsp @ 0x5648cf92fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cf92fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T13:10:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.075, "last_started_at": "2026-07-01T13:10:01+09:00", "pid": 9099, "process": {"exists": false, "pid": 9099}, "restart_count": 39, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.17, "sample_count": 1265, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19474, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:12:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T13:12:27+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T13:12:27+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T13:12:27.268485+09:00", "mtime_epoch": 1782879147.2684846, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197949}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:12:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.93, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 57, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.91, "pushed": 1.91, "read": 1.91}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.224, "count": 57, "max_ms": 11.612, "p95_ms": 10.013}, "gstreamer_push": {"avg_ms": 0.872, "count": 57, "max_ms": 0.946, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.125, "count": 57, "max_ms": 55.868, "p95_ms": 51.506}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.702, "count": 57, "max_ms": 85.042, "p95_ms": 78.894}, "yolo11_640_fire_11class": {"avg_ms": 45.013, "count": 57, "max_ms": 64.374, "p95_ms": 58.115}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.94, "count": 57, "max_ms": 86.177, "p95_ms": 83.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.468, "count": 57, "max_ms": 228.779, "p95_ms": 225.755}, "read_interval": {"avg_ms": 526.158, "count": 57, "max_ms": 10125.916, "p95_ms": 239.94}, "triton_wait": {"avg_ms": 59.971, "count": 57, "max_ms": 86.189, "p95_ms": 83.39}, "visualize": {"avg_ms": 9.731, "count": 57, "max_ms": 11.433, "p95_ms": 10.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.849, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5937.266, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:12:27+09:00", "last_error_summary": "[h264 @ 0x5648d02e3740] left block unavailable for requested intra mode [h264 @ 0x5648d02e3740] error while decoding MB 0 3, bytestream 35811 [h264 @ 0x5648d05502c0] error while decoding MB 25 9, bytestream -5 [in#0/rtsp @ 0x5648cf92fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cf92fe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:12:27+09:00", "last_error_summary": "[h264 @ 0x5648d02e3740] left block unavailable for requested intra mode [h264 @ 0x5648d02e3740] error while decoding MB 0 3, bytestream 35811 [h264 @ 0x5648d05502c0] error while decoding MB 25 9, bytestream -5 [in#0/rtsp @ 0x5648cf92fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cf92fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T13:12:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.232, "last_started_at": "2026-07-01T13:10:01+09:00", "pid": 9099, "process": {"exists": false, "pid": 9099}, "restart_count": 39, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.17, "sample_count": 1265, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19475, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:12:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:12:27+09:00", "last_frame_read_age_sec": 0.206, "last_frame_read_at": "2026-07-01T13:12:27+09:00", "mjpeg": {"age_sec": 0.194, "exists": true, "mtime": "2026-07-01T13:12:27.268485+09:00", "mtime_epoch": 1782879147.2684846, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197949}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:12:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.93, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 57, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.9, "pushed": 1.9, "read": 1.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.234, "count": 57, "max_ms": 11.612, "p95_ms": 10.013}, "gstreamer_push": {"avg_ms": 0.872, "count": 57, "max_ms": 0.946, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.126, "count": 57, "max_ms": 55.868, "p95_ms": 51.506}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.839, "count": 57, "max_ms": 85.042, "p95_ms": 78.894}, "yolo11_640_fire_11class": {"avg_ms": 45.179, "count": 57, "max_ms": 64.374, "p95_ms": 58.115}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.081, "count": 57, "max_ms": 86.177, "p95_ms": 83.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.639, "count": 57, "max_ms": 228.779, "p95_ms": 225.755}, "read_interval": {"avg_ms": 526.158, "count": 57, "max_ms": 10125.916, "p95_ms": 239.94}, "triton_wait": {"avg_ms": 60.112, "count": 57, "max_ms": 86.189, "p95_ms": 83.39}, "visualize": {"avg_ms": 9.737, "count": 57, "max_ms": 11.433, "p95_ms": 10.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5937.258, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:12:27+09:00", "last_error_summary": "[h264 @ 0x5648d02e3740] left block unavailable for requested intra mode [h264 @ 0x5648d02e3740] error while decoding MB 0 3, bytestream 35811 [h264 @ 0x5648d05502c0] error while decoding MB 25 9, bytestream -5 [in#0/rtsp @ 0x5648cf92fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cf92fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T13:12:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 146.235, "last_started_at": "2026-07-01T13:10:01+09:00", "pid": 9099, "process": {"exists": false, "pid": 9099}, "restart_count": 39, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.17, "sample_count": 1265, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19475, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:12:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T13:12:27+09:00", "last_frame_read_age_sec": 0.209, "last_frame_read_at": "2026-07-01T13:12:27+09:00", "mjpeg": {"age_sec": 0.197, "exists": true, "mtime": "2026-07-01T13:12:27.268485+09:00", "mtime_epoch": 1782879147.2684846, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197949}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:12:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.93, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 57, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.9, "pushed": 1.9, "read": 1.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.234, "count": 57, "max_ms": 11.612, "p95_ms": 10.013}, "gstreamer_push": {"avg_ms": 0.872, "count": 57, "max_ms": 0.946, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.126, "count": 57, "max_ms": 55.868, "p95_ms": 51.506}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.839, "count": 57, "max_ms": 85.042, "p95_ms": 78.894}, "yolo11_640_fire_11class": {"avg_ms": 45.179, "count": 57, "max_ms": 64.374, "p95_ms": 58.115}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.081, "count": 57, "max_ms": 86.177, "p95_ms": 83.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.639, "count": 57, "max_ms": 228.779, "p95_ms": 225.755}, "read_interval": {"avg_ms": 526.158, "count": 57, "max_ms": 10125.916, "p95_ms": 239.94}, "triton_wait": {"avg_ms": 60.112, "count": 57, "max_ms": 86.189, "p95_ms": 83.39}, "visualize": {"avg_ms": 9.737, "count": 57, "max_ms": 11.433, "p95_ms": 10.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6045.547, "limit_mb": 8796093022207.996}, "detail": {"pid": 9266, "restart_count": 40, "start_count": 41}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:12:27+09:00", "last_error_summary": "[h264 @ 0x5648d02e3740] left block unavailable for requested intra mode [h264 @ 0x5648d02e3740] error while decoding MB 0 3, bytestream 35811 [h264 @ 0x5648d05502c0] error while decoding MB 25 9, bytestream -5 [in#0/rtsp @ 0x5648cf92fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648cf92fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T13:12:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:12:29+09:00", "pid": 9266, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.125, "name": "ffmpeg", "pid": 9266, "rss_mb": 139.125, "state": "S (sleeping)", "threads": 106}, "restart_count": 40, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.518, "sample_count": 1247, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19475, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:12:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.12, "last_frame_pushed_at": "2026-07-01T13:12:27+09:00", "last_frame_read_age_sec": 4.323, "last_frame_read_at": "2026-07-01T13:12:27+09:00", "mjpeg": {"age_sec": 4.103, "exists": true, "mtime": "2026-07-01T13:12:27.476489+09:00", "mtime_epoch": 1782879147.4764888, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198055}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:12:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.93, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 38, "pushed_frames": 38, "read_frames": 37, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.271, "pushed": 1.271, "read": 1.238}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.077, "count": 37, "max_ms": 11.612, "p95_ms": 10.308}, "gstreamer_push": {"avg_ms": 0.874, "count": 38, "max_ms": 0.946, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.861, "count": 37, "max_ms": 55.868, "p95_ms": 52.846}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.863, "count": 37, "max_ms": 85.042, "p95_ms": 83.329}, "yolo11_640_fire_11class": {"avg_ms": 45.334, "count": 37, "max_ms": 64.374, "p95_ms": 62.457}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.107, "count": 37, "max_ms": 86.177, "p95_ms": 85.693}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.788, "count": 38, "max_ms": 228.779, "p95_ms": 228.518}, "read_interval": {"avg_ms": 696.661, "count": 37, "max_ms": 10125.916, "p95_ms": 8271.85}, "triton_wait": {"avg_ms": 60.147, "count": 37, "max_ms": 86.189, "p95_ms": 85.705}, "visualize": {"avg_ms": 9.717, "count": 37, "max_ms": 11.433, "p95_ms": 10.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5938.59, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55cd60c117c0] error while decoding MB 31 21, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:14:36+09:00", "last_error_summary": "[h264 @ 0x55cd60c117c0] error while decoding MB 31 21, bytestream -10", "last_exit_at": "2026-07-01T13:12:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.893, "last_started_at": "2026-07-01T13:12:29+09:00", "pid": 9266, "process": {"exists": false, "pid": 9266}, "restart_count": 40, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.518, "sample_count": 1265, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20051, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:14:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T13:14:36+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T13:14:36+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T13:14:36.425550+09:00", "mtime_epoch": 1782879276.4255495, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192623}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:14:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.93, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.647, "pushed": 4.647, "read": 4.647}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.445, "count": 138, "max_ms": 11.707, "p95_ms": 10.643}, "gstreamer_push": {"avg_ms": 0.868, "count": 139, "max_ms": 0.968, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.712, "count": 138, "max_ms": 50.896, "p95_ms": 44.845}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.163, "count": 138, "max_ms": 82.21, "p95_ms": 77.197}, "yolo11_640_fire_11class": {"avg_ms": 48.135, "count": 138, "max_ms": 62.539, "p95_ms": 58.868}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.821, "count": 138, "max_ms": 83.441, "p95_ms": 79.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 207.111, "count": 139, "max_ms": 244.382, "p95_ms": 238.667}, "read_interval": {"avg_ms": 216.316, "count": 139, "max_ms": 251.352, "p95_ms": 245.566}, "triton_wait": {"avg_ms": 62.36, "count": 138, "max_ms": 83.453, "p95_ms": 79.511}, "visualize": {"avg_ms": 9.766, "count": 138, "max_ms": 11.297, "p95_ms": 10.957}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5937.887, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:14:36+09:00", "last_error_summary": "[h264 @ 0x55cd60c117c0] error while decoding MB 31 21, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:14:36+09:00", "last_error_summary": "[h264 @ 0x55cd60c117c0] error while decoding MB 31 21, bytestream -10", "last_exit_at": "2026-07-01T13:14:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.053, "last_started_at": "2026-07-01T13:12:29+09:00", "pid": 9266, "process": {"exists": false, "pid": 9266}, "restart_count": 40, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.518, "sample_count": 1266, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20052, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:14:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:14:36+09:00", "last_frame_read_age_sec": 0.213, "last_frame_read_at": "2026-07-01T13:14:36+09:00", "mjpeg": {"age_sec": 0.206, "exists": true, "mtime": "2026-07-01T13:14:36.425550+09:00", "mtime_epoch": 1782879276.4255495, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192623}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:14:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.93, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.654, "pushed": 4.654, "read": 4.62}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.467, "count": 139, "max_ms": 12.409, "p95_ms": 10.702}, "gstreamer_push": {"avg_ms": 0.868, "count": 139, "max_ms": 0.968, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.71, "count": 139, "max_ms": 50.896, "p95_ms": 44.845}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.152, "count": 139, "max_ms": 82.21, "p95_ms": 77.197}, "yolo11_640_fire_11class": {"avg_ms": 48.119, "count": 139, "max_ms": 62.539, "p95_ms": 58.868}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.799, "count": 139, "max_ms": 83.441, "p95_ms": 79.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 207.126, "count": 139, "max_ms": 244.382, "p95_ms": 238.667}, "read_interval": {"avg_ms": 216.317, "count": 138, "max_ms": 251.352, "p95_ms": 245.566}, "triton_wait": {"avg_ms": 62.333, "count": 139, "max_ms": 83.453, "p95_ms": 79.511}, "visualize": {"avg_ms": 9.767, "count": 139, "max_ms": 11.297, "p95_ms": 10.957}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5937.879, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:14:36+09:00", "last_error_summary": "[h264 @ 0x55cd60c117c0] error while decoding MB 31 21, bytestream -10", "last_exit_at": "2026-07-01T13:14:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.057, "last_started_at": "2026-07-01T13:12:29+09:00", "pid": 9266, "process": {"exists": false, "pid": 9266}, "restart_count": 40, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.518, "sample_count": 1265, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20052, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:14:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:14:36+09:00", "last_frame_read_age_sec": 0.217, "last_frame_read_at": "2026-07-01T13:14:36+09:00", "mjpeg": {"age_sec": 0.21, "exists": true, "mtime": "2026-07-01T13:14:36.425550+09:00", "mtime_epoch": 1782879276.4255495, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192623}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:14:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.93, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.653, "pushed": 4.653, "read": 4.62}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.467, "count": 139, "max_ms": 12.409, "p95_ms": 10.702}, "gstreamer_push": {"avg_ms": 0.868, "count": 139, "max_ms": 0.968, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.71, "count": 139, "max_ms": 50.896, "p95_ms": 44.845}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.152, "count": 139, "max_ms": 82.21, "p95_ms": 77.197}, "yolo11_640_fire_11class": {"avg_ms": 48.119, "count": 139, "max_ms": 62.539, "p95_ms": 58.868}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.799, "count": 139, "max_ms": 83.441, "p95_ms": 79.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 207.126, "count": 139, "max_ms": 244.382, "p95_ms": 238.667}, "read_interval": {"avg_ms": 216.317, "count": 138, "max_ms": 251.352, "p95_ms": 245.566}, "triton_wait": {"avg_ms": 62.333, "count": 139, "max_ms": 83.453, "p95_ms": 79.511}, "visualize": {"avg_ms": 9.767, "count": 139, "max_ms": 11.297, "p95_ms": 10.957}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 167}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6046.949, "limit_mb": 8796093022207.996}, "detail": {"pid": 9373, "restart_count": 41, "start_count": 42}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:14:36+09:00", "last_error_summary": "[h264 @ 0x55cd60c117c0] error while decoding MB 31 21, bytestream -10", "last_exit_at": "2026-07-01T13:14:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:14:39+09:00", "pid": 9373, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.898, "name": "ffmpeg", "pid": 9373, "rss_mb": 139.898, "state": "S (sleeping)", "threads": 106}, "restart_count": 41, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.557, "sample_count": 1246, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20052, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:14:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.503, "last_frame_pushed_at": "2026-07-01T13:14:36+09:00", "last_frame_read_age_sec": 4.713, "last_frame_read_at": "2026-07-01T13:14:36+09:00", "mjpeg": {"age_sec": 4.486, "exists": true, "mtime": "2026-07-01T13:14:36.645551+09:00", "mtime_epoch": 1782879276.6455512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:14:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2359.93, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.944, "pushed": 3.944, "read": 3.91}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.608, "count": 117, "max_ms": 12.409, "p95_ms": 10.643}, "gstreamer_push": {"avg_ms": 0.867, "count": 118, "max_ms": 0.938, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.743, "count": 117, "max_ms": 50.896, "p95_ms": 44.845}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.383, "count": 117, "max_ms": 82.21, "p95_ms": 77.646}, "yolo11_640_fire_11class": {"avg_ms": 48.58, "count": 117, "max_ms": 62.539, "p95_ms": 59.282}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.236, "count": 117, "max_ms": 83.441, "p95_ms": 80.563}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.007, "count": 118, "max_ms": 244.382, "p95_ms": 238.795}, "read_interval": {"avg_ms": 217.142, "count": 117, "max_ms": 251.352, "p95_ms": 245.836}, "triton_wait": {"avg_ms": 62.777, "count": 117, "max_ms": 83.453, "p95_ms": 80.654}, "visualize": {"avg_ms": 9.814, "count": 117, "max_ms": 11.297, "p95_ms": 10.917}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 171}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5939.512, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ca8e5684c0] error while decoding MB 39 36, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:16:26+09:00", "last_error_summary": "[h264 @ 0x55ca8e5684c0] error while decoding MB 39 36, bytestream -11", "last_exit_at": "2026-07-01T13:14:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.105, "last_started_at": "2026-07-01T13:14:39+09:00", "pid": 9373, "process": {"exists": false, "pid": 9373}, "restart_count": 41, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.946, "sample_count": 1255, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20524, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:16:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.044, "last_frame_pushed_at": "2026-07-01T13:16:26+09:00", "last_frame_read_age_sec": 0.037, "last_frame_read_at": "2026-07-01T13:16:26+09:00", "mjpeg": {"age_sec": 0.021, "exists": true, "mtime": "2026-07-01T13:16:26.214511+09:00", "mtime_epoch": 1782879386.2145107, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:16:26+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2360.188, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.603, "pushed": 4.603, "read": 4.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.544, "count": 137, "max_ms": 12.793, "p95_ms": 10.517}, "gstreamer_push": {"avg_ms": 0.858, "count": 138, "max_ms": 1.058, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.306, "count": 137, "max_ms": 49.743, "p95_ms": 44.877}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.88, "count": 137, "max_ms": 77.679, "p95_ms": 73.777}, "yolo11_640_fire_11class": {"avg_ms": 48.834, "count": 137, "max_ms": 63.494, "p95_ms": 60.865}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.6, "count": 137, "max_ms": 78.952, "p95_ms": 72.757}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.86, "count": 138, "max_ms": 231.165, "p95_ms": 224.323}, "read_interval": {"avg_ms": 218.593, "count": 138, "max_ms": 241.969, "p95_ms": 233.799}, "triton_wait": {"avg_ms": 63.306, "count": 137, "max_ms": 78.962, "p95_ms": 74.648}, "visualize": {"avg_ms": 9.803, "count": 137, "max_ms": 11.87, "p95_ms": 10.741}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 171}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5938.984, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:16:26+09:00", "last_error_summary": "[h264 @ 0x55ca8e5684c0] error while decoding MB 39 36, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:16:26+09:00", "last_error_summary": "[h264 @ 0x55ca8e5684c0] error while decoding MB 39 36, bytestream -11", "last_exit_at": "2026-07-01T13:16:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.279, "last_started_at": "2026-07-01T13:14:39+09:00", "pid": 9373, "process": {"exists": false, "pid": 9373}, "restart_count": 41, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.946, "sample_count": 1255, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20525, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:16:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:16:26+09:00", "last_frame_read_age_sec": 0.211, "last_frame_read_at": "2026-07-01T13:16:26+09:00", "mjpeg": {"age_sec": 0.195, "exists": true, "mtime": "2026-07-01T13:16:26.214511+09:00", "mtime_epoch": 1782879386.2145107, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:16:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2360.188, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.612, "pushed": 4.612, "read": 4.578}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.535, "count": 137, "max_ms": 12.793, "p95_ms": 10.517}, "gstreamer_push": {"avg_ms": 0.858, "count": 138, "max_ms": 1.058, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.315, "count": 137, "max_ms": 49.743, "p95_ms": 44.877}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.884, "count": 137, "max_ms": 77.679, "p95_ms": 73.777}, "yolo11_640_fire_11class": {"avg_ms": 48.814, "count": 137, "max_ms": 63.494, "p95_ms": 60.865}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.605, "count": 137, "max_ms": 78.952, "p95_ms": 72.757}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.738, "count": 138, "max_ms": 231.165, "p95_ms": 224.323}, "read_interval": {"avg_ms": 218.492, "count": 137, "max_ms": 241.969, "p95_ms": 233.799}, "triton_wait": {"avg_ms": 63.31, "count": 137, "max_ms": 78.962, "p95_ms": 74.648}, "visualize": {"avg_ms": 9.799, "count": 137, "max_ms": 11.87, "p95_ms": 10.741}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 171}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5939.102, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:16:26+09:00", "last_error_summary": "[h264 @ 0x55ca8e5684c0] error while decoding MB 39 36, bytestream -11", "last_exit_at": "2026-07-01T13:16:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.283, "last_started_at": "2026-07-01T13:14:39+09:00", "pid": 9373, "process": {"exists": false, "pid": 9373}, "restart_count": 41, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.946, "sample_count": 1255, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20525, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:16:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:16:26+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T13:16:26+09:00", "mjpeg": {"age_sec": 0.198, "exists": true, "mtime": "2026-07-01T13:16:26.214511+09:00", "mtime_epoch": 1782879386.2145107, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:16:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2360.188, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.611, "pushed": 4.611, "read": 4.578}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.535, "count": 137, "max_ms": 12.793, "p95_ms": 10.517}, "gstreamer_push": {"avg_ms": 0.858, "count": 138, "max_ms": 1.058, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.315, "count": 137, "max_ms": 49.743, "p95_ms": 44.877}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.884, "count": 137, "max_ms": 77.679, "p95_ms": 73.777}, "yolo11_640_fire_11class": {"avg_ms": 48.814, "count": 137, "max_ms": 63.494, "p95_ms": 60.865}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.605, "count": 137, "max_ms": 78.952, "p95_ms": 72.757}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.738, "count": 138, "max_ms": 231.165, "p95_ms": 224.323}, "read_interval": {"avg_ms": 218.492, "count": 137, "max_ms": 241.969, "p95_ms": 233.799}, "triton_wait": {"avg_ms": 63.31, "count": 137, "max_ms": 78.962, "p95_ms": 74.648}, "visualize": {"avg_ms": 9.799, "count": 137, "max_ms": 11.87, "p95_ms": 10.741}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 171}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6047.047, "limit_mb": 8796093022207.996}, "detail": {"pid": 9480, "restart_count": 42, "start_count": 43}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:16:26+09:00", "last_error_summary": "[h264 @ 0x55ca8e5684c0] error while decoding MB 39 36, bytestream -11", "last_exit_at": "2026-07-01T13:16:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:16:29+09:00", "pid": 9480, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.863, "name": "ffmpeg", "pid": 9480, "rss_mb": 138.863, "state": "S (sleeping)", "threads": 106}, "restart_count": 42, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.043, "sample_count": 1233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20525, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:16:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.671, "last_frame_pushed_at": "2026-07-01T13:16:26+09:00", "last_frame_read_age_sec": 4.878, "last_frame_read_at": "2026-07-01T13:16:26+09:00", "mjpeg": {"age_sec": 4.646, "exists": true, "mtime": "2026-07-01T13:16:26.430513+09:00", "mtime_epoch": 1782879386.4305127, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186845}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:16:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2360.188, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.867, "pushed": 3.9, "read": 3.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.5, "count": 116, "max_ms": 12.793, "p95_ms": 10.458}, "gstreamer_push": {"avg_ms": 0.86, "count": 117, "max_ms": 1.058, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.996, "count": 116, "max_ms": 49.743, "p95_ms": 43.979}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.155, "count": 116, "max_ms": 77.679, "p95_ms": 73.777}, "yolo11_640_fire_11class": {"avg_ms": 49.214, "count": 116, "max_ms": 63.494, "p95_ms": 60.865}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.648, "count": 116, "max_ms": 78.952, "p95_ms": 72.39}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.599, "count": 116, "max_ms": 231.165, "p95_ms": 224.039}, "read_interval": {"avg_ms": 218.316, "count": 116, "max_ms": 241.969, "p95_ms": 233.583}, "triton_wait": {"avg_ms": 63.437, "count": 116, "max_ms": 78.962, "p95_ms": 74.648}, "visualize": {"avg_ms": 9.797, "count": 116, "max_ms": 11.87, "p95_ms": 10.878}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 175}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5940.52, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:16:26+09:00", "last_error_summary": "[h264 @ 0x55ca8e5684c0] error while decoding MB 39 36, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:16:26+09:00", "last_error_summary": "[h264 @ 0x55ca8e5684c0] error while decoding MB 39 36, bytestream -11", "last_exit_at": "2026-07-01T13:18:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.13, "last_started_at": "2026-07-01T13:16:29+09:00", "pid": 9480, "process": {"exists": false, "pid": 9480}, "restart_count": 42, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 242.292, "sample_count": 1295, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21103, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:18:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T13:18:44+09:00", "last_frame_read_age_sec": 0.242, "last_frame_read_at": "2026-07-01T13:18:43+09:00", "mjpeg": {"age_sec": 0.234, "exists": true, "mtime": "2026-07-01T13:18:43.971986+09:00", "mtime_epoch": 1782879523.9719856, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194034}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:18:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2360.445, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.094, "pushed": 4.094, "read": 4.06}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.726, "count": 122, "max_ms": 14.394, "p95_ms": 12.755}, "gstreamer_push": {"avg_ms": 0.859, "count": 122, "max_ms": 0.941, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.599, "count": 122, "max_ms": 59.227, "p95_ms": 46.494}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.958, "count": 122, "max_ms": 82.896, "p95_ms": 80.725}, "yolo11_640_fire_11class": {"avg_ms": 50.07, "count": 122, "max_ms": 63.104, "p95_ms": 60.734}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.864, "count": 122, "max_ms": 81.348, "p95_ms": 79.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.461, "count": 122, "max_ms": 266.718, "p95_ms": 256.656}, "read_interval": {"avg_ms": 246.241, "count": 121, "max_ms": 276.262, "p95_ms": 266.477}, "triton_wait": {"avg_ms": 68.702, "count": 122, "max_ms": 82.908, "p95_ms": 80.737}, "visualize": {"avg_ms": 11.026, "count": 122, "max_ms": 15.301, "p95_ms": 13.438}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.801, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 175}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5940.594, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:16:26+09:00", "last_error_summary": "[h264 @ 0x55ca8e5684c0] error while decoding MB 39 36, bytestream -11", "last_exit_at": "2026-07-01T13:18:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.133, "last_started_at": "2026-07-01T13:16:29+09:00", "pid": 9480, "process": {"exists": false, "pid": 9480}, "restart_count": 42, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 242.292, "sample_count": 1295, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21103, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:18:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T13:18:44+09:00", "last_frame_read_age_sec": 0.245, "last_frame_read_at": "2026-07-01T13:18:43+09:00", "mjpeg": {"age_sec": 0.237, "exists": true, "mtime": "2026-07-01T13:18:43.971986+09:00", "mtime_epoch": 1782879523.9719856, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194034}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:18:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2360.445, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.093, "pushed": 4.093, "read": 4.06}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.726, "count": 122, "max_ms": 14.394, "p95_ms": 12.755}, "gstreamer_push": {"avg_ms": 0.859, "count": 122, "max_ms": 0.941, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.599, "count": 122, "max_ms": 59.227, "p95_ms": 46.494}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.958, "count": 122, "max_ms": 82.896, "p95_ms": 80.725}, "yolo11_640_fire_11class": {"avg_ms": 50.07, "count": 122, "max_ms": 63.104, "p95_ms": 60.734}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.864, "count": 122, "max_ms": 81.348, "p95_ms": 79.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.461, "count": 122, "max_ms": 266.718, "p95_ms": 256.656}, "read_interval": {"avg_ms": 246.241, "count": 121, "max_ms": 276.262, "p95_ms": 266.477}, "triton_wait": {"avg_ms": 68.702, "count": 122, "max_ms": 82.908, "p95_ms": 80.737}, "visualize": {"avg_ms": 11.026, "count": 122, "max_ms": 15.301, "p95_ms": 13.438}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.804, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 175}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.938, "limit_mb": 8796093022207.996}, "detail": {"pid": 9587, "restart_count": 43, "start_count": 44}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:16:26+09:00", "last_error_summary": "[h264 @ 0x55ca8e5684c0] error while decoding MB 39 36, bytestream -11", "last_exit_at": "2026-07-01T13:18:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:18:46+09:00", "pid": 9587, "process": {"exists": true, "fd_count": 4, "hwm_mb": 53.18, "name": "ffmpeg", "pid": 9587, "rss_mb": 49.039, "state": "R (running)", "threads": 1}, "restart_count": 43, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 242.379, "sample_count": 1274, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21103, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:18:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.63, "last_frame_pushed_at": "2026-07-01T13:18:44+09:00", "last_frame_read_age_sec": 4.867, "last_frame_read_at": "2026-07-01T13:18:43+09:00", "mjpeg": {"age_sec": 4.615, "exists": true, "mtime": "2026-07-01T13:18:44.215988+09:00", "mtime_epoch": 1782879524.2159884, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194063}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:18:48+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2360.445, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.449, "pushed": 3.449, "read": 3.416}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.752, "count": 102, "max_ms": 14.394, "p95_ms": 12.764}, "gstreamer_push": {"avg_ms": 0.859, "count": 103, "max_ms": 0.941, "p95_ms": 0.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.509, "count": 102, "max_ms": 59.227, "p95_ms": 45.547}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.65, "count": 102, "max_ms": 82.472, "p95_ms": 80.725}, "yolo11_640_fire_11class": {"avg_ms": 49.873, "count": 102, "max_ms": 63.104, "p95_ms": 60.733}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.678, "count": 102, "max_ms": 80.577, "p95_ms": 79.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.449, "count": 103, "max_ms": 266.718, "p95_ms": 257.238}, "read_interval": {"avg_ms": 247.229, "count": 102, "max_ms": 276.262, "p95_ms": 267.314}, "triton_wait": {"avg_ms": 68.476, "count": 102, "max_ms": 82.484, "p95_ms": 80.737}, "visualize": {"avg_ms": 11.124, "count": 102, "max_ms": 15.301, "p95_ms": 13.527}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.862, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5941.84, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55617706c140] error while decoding MB 28 35, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:20:35+09:00", "last_error_summary": "[h264 @ 0x55617706c140] error while decoding MB 28 35, bytestream -7", "last_exit_at": "2026-07-01T13:18:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.859, "last_started_at": "2026-07-01T13:18:46+09:00", "pid": 9587, "process": {"exists": false, "pid": 9587}, "restart_count": 43, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.676, "sample_count": 1252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21547, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:20:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T13:20:35+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T13:20:35+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T13:20:35.645355+09:00", "mtime_epoch": 1782879635.645355, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198038}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:20:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2360.703, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.214, "pushed": 4.214, "read": 4.214}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.452, "count": 125, "max_ms": 11.78, "p95_ms": 11.26}, "gstreamer_push": {"avg_ms": 0.849, "count": 126, "max_ms": 0.92, "p95_ms": 0.882}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.927, "count": 125, "max_ms": 46.299, "p95_ms": 41.195}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.246, "count": 125, "max_ms": 81.513, "p95_ms": 75.655}, "yolo11_640_fire_11class": {"avg_ms": 47.926, "count": 125, "max_ms": 63.553, "p95_ms": 59.201}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.462, "count": 125, "max_ms": 82.722, "p95_ms": 77.643}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.031, "count": 126, "max_ms": 260.566, "p95_ms": 255.951}, "read_interval": {"avg_ms": 238.621, "count": 126, "max_ms": 270.713, "p95_ms": 265.332}, "triton_wait": {"avg_ms": 64.261, "count": 125, "max_ms": 82.734, "p95_ms": 77.652}, "visualize": {"avg_ms": 10.748, "count": 125, "max_ms": 12.128, "p95_ms": 11.684}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5941.258, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:20:35+09:00", "last_error_summary": "[h264 @ 0x55617706c140] error while decoding MB 28 35, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:20:35+09:00", "last_error_summary": "[h264 @ 0x55617706c140] error while decoding MB 28 35, bytestream -7", "last_exit_at": "2026-07-01T13:20:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.039, "last_started_at": "2026-07-01T13:18:46+09:00", "pid": 9587, "process": {"exists": false, "pid": 9587}, "restart_count": 43, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.676, "sample_count": 1252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21548, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:20:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T13:20:35+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T13:20:35+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T13:20:35.645355+09:00", "mtime_epoch": 1782879635.645355, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198038}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:20:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2360.703, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.221, "pushed": 4.221, "read": 4.187}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.452, "count": 126, "max_ms": 11.78, "p95_ms": 11.26}, "gstreamer_push": {"avg_ms": 0.85, "count": 126, "max_ms": 0.92, "p95_ms": 0.882}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.913, "count": 126, "max_ms": 46.299, "p95_ms": 41.195}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.25, "count": 126, "max_ms": 81.513, "p95_ms": 75.655}, "yolo11_640_fire_11class": {"avg_ms": 47.905, "count": 126, "max_ms": 63.553, "p95_ms": 59.201}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.455, "count": 126, "max_ms": 82.722, "p95_ms": 77.643}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.084, "count": 126, "max_ms": 260.566, "p95_ms": 255.951}, "read_interval": {"avg_ms": 238.699, "count": 125, "max_ms": 270.713, "p95_ms": 265.332}, "triton_wait": {"avg_ms": 64.257, "count": 126, "max_ms": 82.734, "p95_ms": 77.652}, "visualize": {"avg_ms": 10.746, "count": 126, "max_ms": 12.128, "p95_ms": 11.684}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.853, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5941.25, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:20:35+09:00", "last_error_summary": "[h264 @ 0x55617706c140] error while decoding MB 28 35, bytestream -7", "last_exit_at": "2026-07-01T13:20:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.043, "last_started_at": "2026-07-01T13:18:46+09:00", "pid": 9587, "process": {"exists": false, "pid": 9587}, "restart_count": 43, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.676, "sample_count": 1252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21548, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:20:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T13:20:35+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T13:20:35+09:00", "mjpeg": {"age_sec": 0.227, "exists": true, "mtime": "2026-07-01T13:20:35.645355+09:00", "mtime_epoch": 1782879635.645355, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198038}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:20:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2360.703, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.22, "pushed": 4.22, "read": 4.187}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.452, "count": 126, "max_ms": 11.78, "p95_ms": 11.26}, "gstreamer_push": {"avg_ms": 0.85, "count": 126, "max_ms": 0.92, "p95_ms": 0.882}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.913, "count": 126, "max_ms": 46.299, "p95_ms": 41.195}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.25, "count": 126, "max_ms": 81.513, "p95_ms": 75.655}, "yolo11_640_fire_11class": {"avg_ms": 47.905, "count": 126, "max_ms": 63.553, "p95_ms": 59.201}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.455, "count": 126, "max_ms": 82.722, "p95_ms": 77.643}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.084, "count": 126, "max_ms": 260.566, "p95_ms": 255.951}, "read_interval": {"avg_ms": 238.699, "count": 125, "max_ms": 270.713, "p95_ms": 265.332}, "triton_wait": {"avg_ms": 64.257, "count": 126, "max_ms": 82.734, "p95_ms": 77.652}, "visualize": {"avg_ms": 10.746, "count": 126, "max_ms": 12.128, "p95_ms": 11.684}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.856, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6049.488, "limit_mb": 8796093022207.996}, "detail": {"pid": 9694, "restart_count": 44, "start_count": 45}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:20:35+09:00", "last_error_summary": "[h264 @ 0x55617706c140] error while decoding MB 28 35, bytestream -7", "last_exit_at": "2026-07-01T13:20:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:20:37+09:00", "pid": 9694, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.859, "name": "ffmpeg", "pid": 9694, "rss_mb": 138.859, "state": "S (sleeping)", "threads": 106}, "restart_count": 44, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.676, "sample_count": 1235, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21548, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:20:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.03, "last_frame_pushed_at": "2026-07-01T13:20:35+09:00", "last_frame_read_age_sec": 4.256, "last_frame_read_at": "2026-07-01T13:20:35+09:00", "mjpeg": {"age_sec": 4.014, "exists": true, "mtime": "2026-07-01T13:20:35.881358+09:00", "mtime_epoch": 1782879635.881358, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198713}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:20:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2360.703, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.643, "pushed": 3.643, "read": 3.61}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.468, "count": 108, "max_ms": 11.78, "p95_ms": 11.305}, "gstreamer_push": {"avg_ms": 0.85, "count": 109, "max_ms": 0.92, "p95_ms": 0.882}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.145, "count": 108, "max_ms": 46.299, "p95_ms": 41.21}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.526, "count": 108, "max_ms": 81.513, "p95_ms": 76.912}, "yolo11_640_fire_11class": {"avg_ms": 48.345, "count": 108, "max_ms": 63.553, "p95_ms": 59.25}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.869, "count": 108, "max_ms": 82.722, "p95_ms": 78.325}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.072, "count": 109, "max_ms": 260.566, "p95_ms": 256.112}, "read_interval": {"avg_ms": 239.69, "count": 108, "max_ms": 270.713, "p95_ms": 265.396}, "triton_wait": {"avg_ms": 64.589, "count": 108, "max_ms": 82.734, "p95_ms": 78.673}, "visualize": {"avg_ms": 10.775, "count": 108, "max_ms": 12.128, "p95_ms": 11.775}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 184}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5943.301, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55aca0693680] error while decoding MB 64 34, bytestream -30", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:22:46+09:00", "last_error_summary": "[h264 @ 0x55aca0693680] error while decoding MB 64 34, bytestream -30", "last_exit_at": "2026-07-01T13:20:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.833, "last_started_at": "2026-07-01T13:20:37+09:00", "pid": 9694, "process": {"exists": false, "pid": 9694}, "restart_count": 44, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.895, "sample_count": 1248, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22110, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:22:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T13:22:46+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T13:22:46+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T13:22:46.687117+09:00", "mtime_epoch": 1782879766.6871166, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193683}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:22:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.219, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.352, "pushed": 4.352, "read": 4.352}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.927, "count": 130, "max_ms": 12.994, "p95_ms": 10.737}, "gstreamer_push": {"avg_ms": 0.84, "count": 130, "max_ms": 0.914, "p95_ms": 0.873}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.356, "count": 130, "max_ms": 48.509, "p95_ms": 44.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.477, "count": 130, "max_ms": 85.484, "p95_ms": 76.882}, "yolo11_640_fire_11class": {"avg_ms": 49.743, "count": 130, "max_ms": 69.22, "p95_ms": 63.476}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.95, "count": 130, "max_ms": 83.674, "p95_ms": 80.21}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.262, "count": 130, "max_ms": 254.555, "p95_ms": 244.024}, "read_interval": {"avg_ms": 230.891, "count": 130, "max_ms": 263.716, "p95_ms": 253.78}, "triton_wait": {"avg_ms": 64.243, "count": 130, "max_ms": 85.496, "p95_ms": 80.884}, "visualize": {"avg_ms": 10.471, "count": 130, "max_ms": 13.79, "p95_ms": 11.119}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 184}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5942.992, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:22:46+09:00", "last_error_summary": "[h264 @ 0x55aca0693680] error while decoding MB 64 34, bytestream -30", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:22:46+09:00", "last_error_summary": "[h264 @ 0x55aca0693680] error while decoding MB 64 34, bytestream -30", "last_exit_at": "2026-07-01T13:22:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.996, "last_started_at": "2026-07-01T13:20:37+09:00", "pid": 9694, "process": {"exists": false, "pid": 9694}, "restart_count": 44, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.895, "sample_count": 1249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22111, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:22:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:22:46+09:00", "last_frame_read_age_sec": 0.213, "last_frame_read_at": "2026-07-01T13:22:46+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-01T13:22:46.687117+09:00", "mtime_epoch": 1782879766.6871166, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193683}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:22:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.358, "pushed": 4.358, "read": 4.325}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.947, "count": 130, "max_ms": 12.994, "p95_ms": 10.834}, "gstreamer_push": {"avg_ms": 0.84, "count": 130, "max_ms": 0.914, "p95_ms": 0.873}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.334, "count": 130, "max_ms": 48.509, "p95_ms": 44.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.478, "count": 130, "max_ms": 85.484, "p95_ms": 76.882}, "yolo11_640_fire_11class": {"avg_ms": 49.747, "count": 130, "max_ms": 69.22, "p95_ms": 63.476}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.944, "count": 130, "max_ms": 83.674, "p95_ms": 80.21}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.357, "count": 130, "max_ms": 254.555, "p95_ms": 244.024}, "read_interval": {"avg_ms": 231.085, "count": 129, "max_ms": 263.716, "p95_ms": 253.78}, "triton_wait": {"avg_ms": 64.237, "count": 130, "max_ms": 85.496, "p95_ms": 80.884}, "visualize": {"avg_ms": 10.484, "count": 130, "max_ms": 13.79, "p95_ms": 11.142}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.83, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 184}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5942.984, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:22:46+09:00", "last_error_summary": "[h264 @ 0x55aca0693680] error while decoding MB 64 34, bytestream -30", "last_exit_at": "2026-07-01T13:22:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.999, "last_started_at": "2026-07-01T13:20:37+09:00", "pid": 9694, "process": {"exists": false, "pid": 9694}, "restart_count": 44, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.895, "sample_count": 1249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22111, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:22:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T13:22:46+09:00", "last_frame_read_age_sec": 0.216, "last_frame_read_at": "2026-07-01T13:22:46+09:00", "mjpeg": {"age_sec": 0.206, "exists": true, "mtime": "2026-07-01T13:22:46.687117+09:00", "mtime_epoch": 1782879766.6871166, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193683}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:22:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.358, "pushed": 4.358, "read": 4.324}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.947, "count": 130, "max_ms": 12.994, "p95_ms": 10.834}, "gstreamer_push": {"avg_ms": 0.84, "count": 130, "max_ms": 0.914, "p95_ms": 0.873}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.334, "count": 130, "max_ms": 48.509, "p95_ms": 44.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.478, "count": 130, "max_ms": 85.484, "p95_ms": 76.882}, "yolo11_640_fire_11class": {"avg_ms": 49.747, "count": 130, "max_ms": 69.22, "p95_ms": 63.476}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.944, "count": 130, "max_ms": 83.674, "p95_ms": 80.21}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.357, "count": 130, "max_ms": 254.555, "p95_ms": 244.024}, "read_interval": {"avg_ms": 231.085, "count": 129, "max_ms": 263.716, "p95_ms": 253.78}, "triton_wait": {"avg_ms": 64.237, "count": 130, "max_ms": 85.496, "p95_ms": 80.884}, "visualize": {"avg_ms": 10.484, "count": 130, "max_ms": 13.79, "p95_ms": 11.142}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.833, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 184}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6051.219, "limit_mb": 8796093022207.996}, "detail": {"pid": 9801, "restart_count": 45, "start_count": 46}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:22:46+09:00", "last_error_summary": "[h264 @ 0x55aca0693680] error while decoding MB 64 34, bytestream -30", "last_exit_at": "2026-07-01T13:22:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:22:49+09:00", "pid": 9801, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.949, "name": "ffmpeg", "pid": 9801, "rss_mb": 138.949, "state": "S (sleeping)", "threads": 106}, "restart_count": 45, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 247.042, "sample_count": 1229, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22111, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:22:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.271, "last_frame_pushed_at": "2026-07-01T13:22:46+09:00", "last_frame_read_age_sec": 4.481, "last_frame_read_at": "2026-07-01T13:22:46+09:00", "mjpeg": {"age_sec": 4.255, "exists": true, "mtime": "2026-07-01T13:22:46.903120+09:00", "mtime_epoch": 1782879766.9031196, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193960}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:22:51+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.219, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.692, "pushed": 3.692, "read": 3.658}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.052, "count": 110, "max_ms": 11.414, "p95_ms": 10.834}, "gstreamer_push": {"avg_ms": 0.839, "count": 110, "max_ms": 0.914, "p95_ms": 0.873}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.289, "count": 110, "max_ms": 47.637, "p95_ms": 44.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.138, "count": 110, "max_ms": 85.484, "p95_ms": 76.916}, "yolo11_640_fire_11class": {"avg_ms": 49.994, "count": 110, "max_ms": 69.22, "p95_ms": 63.476}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.57, "count": 110, "max_ms": 83.674, "p95_ms": 80.455}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.533, "count": 110, "max_ms": 254.555, "p95_ms": 244.515}, "read_interval": {"avg_ms": 234.304, "count": 109, "max_ms": 263.716, "p95_ms": 254.179}, "triton_wait": {"avg_ms": 64.914, "count": 110, "max_ms": 85.496, "p95_ms": 80.927}, "visualize": {"avg_ms": 10.597, "count": 110, "max_ms": 13.79, "p95_ms": 11.167}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.795, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 189}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5943.809, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:22:46+09:00", "last_error_summary": "[h264 @ 0x55aca0693680] error while decoding MB 64 34, bytestream -30", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:22:46+09:00", "last_error_summary": "[h264 @ 0x55aca0693680] error while decoding MB 64 34, bytestream -30", "last_exit_at": "2026-07-01T13:24:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.959, "last_started_at": "2026-07-01T13:22:49+09:00", "pid": 9801, "process": {"exists": false, "pid": 9801}, "restart_count": 45, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.913, "sample_count": 1313, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22706, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:24:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:24:59+09:00", "last_frame_read_age_sec": 0.202, "last_frame_read_at": "2026-07-01T13:24:58+09:00", "mjpeg": {"age_sec": 0.195, "exists": true, "mtime": "2026-07-01T13:24:58.921027+09:00", "mtime_epoch": 1782879898.9210272, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200041}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:24:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.789, "pushed": 4.789, "read": 4.755}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.104, "count": 143, "max_ms": 15.96, "p95_ms": 9.525}, "gstreamer_push": {"avg_ms": 0.851, "count": 143, "max_ms": 1.593, "p95_ms": 0.881}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.183, "count": 143, "max_ms": 50.84, "p95_ms": 43.679}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.356, "count": 143, "max_ms": 76.694, "p95_ms": 72.719}, "yolo11_640_fire_11class": {"avg_ms": 46.329, "count": 143, "max_ms": 66.084, "p95_ms": 60.451}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.987, "count": 143, "max_ms": 80.838, "p95_ms": 72.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.704, "count": 143, "max_ms": 227.038, "p95_ms": 215.891}, "read_interval": {"avg_ms": 210.315, "count": 142, "max_ms": 238.698, "p95_ms": 224.47}, "triton_wait": {"avg_ms": 60.453, "count": 143, "max_ms": 80.85, "p95_ms": 73.778}, "visualize": {"avg_ms": 9.604, "count": 143, "max_ms": 12.603, "p95_ms": 10.039}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.861, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 189}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5943.777, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:22:46+09:00", "last_error_summary": "[h264 @ 0x55aca0693680] error while decoding MB 64 34, bytestream -30", "last_exit_at": "2026-07-01T13:24:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.963, "last_started_at": "2026-07-01T13:22:49+09:00", "pid": 9801, "process": {"exists": false, "pid": 9801}, "restart_count": 45, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.913, "sample_count": 1313, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22706, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:24:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:24:59+09:00", "last_frame_read_age_sec": 0.206, "last_frame_read_at": "2026-07-01T13:24:58+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T13:24:58.921027+09:00", "mtime_epoch": 1782879898.9210272, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200041}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:24:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.788, "pushed": 4.788, "read": 4.755}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.104, "count": 143, "max_ms": 15.96, "p95_ms": 9.525}, "gstreamer_push": {"avg_ms": 0.851, "count": 143, "max_ms": 1.593, "p95_ms": 0.881}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.183, "count": 143, "max_ms": 50.84, "p95_ms": 43.679}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.356, "count": 143, "max_ms": 76.694, "p95_ms": 72.719}, "yolo11_640_fire_11class": {"avg_ms": 46.329, "count": 143, "max_ms": 66.084, "p95_ms": 60.451}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.987, "count": 143, "max_ms": 80.838, "p95_ms": 72.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.704, "count": 143, "max_ms": 227.038, "p95_ms": 215.891}, "read_interval": {"avg_ms": 210.315, "count": 142, "max_ms": 238.698, "p95_ms": 224.47}, "triton_wait": {"avg_ms": 60.453, "count": 143, "max_ms": 80.85, "p95_ms": 73.778}, "visualize": {"avg_ms": 9.604, "count": 143, "max_ms": 12.603, "p95_ms": 10.039}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.865, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 189}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6062.641, "limit_mb": 8796093022207.996}, "detail": {"pid": 9908, "restart_count": 46, "start_count": 47}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:22:46+09:00", "last_error_summary": "[h264 @ 0x55aca0693680] error while decoding MB 64 34, bytestream -30", "last_exit_at": "2026-07-01T13:25:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:25:01+09:00", "pid": 9908, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.73, "name": "ffmpeg", "pid": 9908, "rss_mb": 149.73, "state": "S (sleeping)", "threads": 106}, "restart_count": 46, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.69, "sample_count": 1294, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22706, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:24:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.461, "last_frame_pushed_at": "2026-07-01T13:24:59+09:00", "last_frame_read_age_sec": 4.66, "last_frame_read_at": "2026-07-01T13:24:58+09:00", "mjpeg": {"age_sec": 4.445, "exists": true, "mtime": "2026-07-01T13:24:59.129030+09:00", "mtime_epoch": 1782879899.1290302, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200257}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:25:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.219, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.067, "pushed": 4.067, "read": 4.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.107, "count": 122, "max_ms": 12.223, "p95_ms": 9.525}, "gstreamer_push": {"avg_ms": 0.845, "count": 122, "max_ms": 0.911, "p95_ms": 0.88}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.517, "count": 122, "max_ms": 50.84, "p95_ms": 43.853}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.837, "count": 122, "max_ms": 76.472, "p95_ms": 72.522}, "yolo11_640_fire_11class": {"avg_ms": 46.356, "count": 122, "max_ms": 66.084, "p95_ms": 61.648}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.847, "count": 122, "max_ms": 80.838, "p95_ms": 72.493}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.037, "count": 122, "max_ms": 222.265, "p95_ms": 215.45}, "read_interval": {"avg_ms": 209.372, "count": 122, "max_ms": 231.618, "p95_ms": 222.482}, "triton_wait": {"avg_ms": 60.099, "count": 122, "max_ms": 80.85, "p95_ms": 73.605}, "visualize": {"avg_ms": 9.559, "count": 122, "max_ms": 11.075, "p95_ms": 9.982}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5944.906, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561a20db9880] error while decoding MB 78 23, bytestream -28", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:27:08+09:00", "last_error_summary": "[h264 @ 0x561a20db9880] error while decoding MB 78 23, bytestream -28", "last_exit_at": "2026-07-01T13:25:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.416, "last_started_at": "2026-07-01T13:25:01+09:00", "pid": 9908, "process": {"exists": false, "pid": 9908}, "restart_count": 46, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.402, "sample_count": 1349, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:27:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T13:27:08+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T13:27:08+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T13:27:08.947006+09:00", "mtime_epoch": 1782880028.9470065, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200935}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:27:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.477, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.68, "pushed": 4.68, "read": 4.68}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.297, "count": 139, "max_ms": 10.627, "p95_ms": 10.158}, "gstreamer_push": {"avg_ms": 0.858, "count": 140, "max_ms": 1.087, "p95_ms": 0.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.032, "count": 139, "max_ms": 48.698, "p95_ms": 40.462}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.231, "count": 139, "max_ms": 77.053, "p95_ms": 71.848}, "yolo11_640_fire_11class": {"avg_ms": 46.362, "count": 139, "max_ms": 66.862, "p95_ms": 57.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.925, "count": 139, "max_ms": 80.519, "p95_ms": 72.885}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.838, "count": 140, "max_ms": 236.497, "p95_ms": 223.434}, "read_interval": {"avg_ms": 214.744, "count": 140, "max_ms": 245.973, "p95_ms": 233.203}, "triton_wait": {"avg_ms": 61.183, "count": 139, "max_ms": 80.531, "p95_ms": 73.913}, "visualize": {"avg_ms": 9.75, "count": 139, "max_ms": 11.119, "p95_ms": 10.743}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5944.828, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:27:08+09:00", "last_error_summary": "[h264 @ 0x561a20db9880] error while decoding MB 78 23, bytestream -28", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:27:08+09:00", "last_error_summary": "[h264 @ 0x561a20db9880] error while decoding MB 78 23, bytestream -28", "last_exit_at": "2026-07-01T13:27:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.573, "last_started_at": "2026-07-01T13:25:01+09:00", "pid": 9908, "process": {"exists": false, "pid": 9908}, "restart_count": 46, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.402, "sample_count": 1349, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:27:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:27:09+09:00", "last_frame_read_age_sec": 0.209, "last_frame_read_at": "2026-07-01T13:27:08+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T13:27:08.947006+09:00", "mtime_epoch": 1782880028.9470065, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200935}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:27:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.477, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.693, "pushed": 4.693, "read": 4.659}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.319, "count": 140, "max_ms": 12.343, "p95_ms": 10.158}, "gstreamer_push": {"avg_ms": 0.858, "count": 140, "max_ms": 1.087, "p95_ms": 0.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.026, "count": 140, "max_ms": 48.698, "p95_ms": 40.174}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.214, "count": 140, "max_ms": 77.053, "p95_ms": 71.536}, "yolo11_640_fire_11class": {"avg_ms": 46.367, "count": 140, "max_ms": 66.862, "p95_ms": 55.562}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.916, "count": 140, "max_ms": 80.519, "p95_ms": 72.874}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.864, "count": 140, "max_ms": 236.497, "p95_ms": 223.434}, "read_interval": {"avg_ms": 214.774, "count": 139, "max_ms": 245.973, "p95_ms": 234.098}, "triton_wait": {"avg_ms": 61.171, "count": 140, "max_ms": 80.531, "p95_ms": 73.196}, "visualize": {"avg_ms": 9.749, "count": 140, "max_ms": 11.119, "p95_ms": 10.689}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.835, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5944.844, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:27:08+09:00", "last_error_summary": "[h264 @ 0x561a20db9880] error while decoding MB 78 23, bytestream -28", "last_exit_at": "2026-07-01T13:27:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.576, "last_started_at": "2026-07-01T13:25:01+09:00", "pid": 9908, "process": {"exists": false, "pid": 9908}, "restart_count": 46, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.402, "sample_count": 1349, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:27:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T13:27:09+09:00", "last_frame_read_age_sec": 0.212, "last_frame_read_at": "2026-07-01T13:27:08+09:00", "mjpeg": {"age_sec": 0.202, "exists": true, "mtime": "2026-07-01T13:27:08.947006+09:00", "mtime_epoch": 1782880028.9470065, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200935}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:27:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.477, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.692, "pushed": 4.692, "read": 4.658}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.319, "count": 140, "max_ms": 12.343, "p95_ms": 10.158}, "gstreamer_push": {"avg_ms": 0.858, "count": 140, "max_ms": 1.087, "p95_ms": 0.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.026, "count": 140, "max_ms": 48.698, "p95_ms": 40.174}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.214, "count": 140, "max_ms": 77.053, "p95_ms": 71.536}, "yolo11_640_fire_11class": {"avg_ms": 46.367, "count": 140, "max_ms": 66.862, "p95_ms": 55.562}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.916, "count": 140, "max_ms": 80.519, "p95_ms": 72.874}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.864, "count": 140, "max_ms": 236.497, "p95_ms": 223.434}, "read_interval": {"avg_ms": 214.774, "count": 139, "max_ms": 245.973, "p95_ms": 234.098}, "triton_wait": {"avg_ms": 61.171, "count": 140, "max_ms": 80.531, "p95_ms": 73.196}, "visualize": {"avg_ms": 9.749, "count": 140, "max_ms": 11.119, "p95_ms": 10.689}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.838, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6052.934, "limit_mb": 8796093022207.996}, "detail": {"pid": 10015, "restart_count": 47, "start_count": 48}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:27:08+09:00", "last_error_summary": "[h264 @ 0x561a20db9880] error while decoding MB 78 23, bytestream -28", "last_exit_at": "2026-07-01T13:27:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:27:11+09:00", "pid": 10015, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.461, "name": "ffmpeg", "pid": 10015, "rss_mb": 137.461, "state": "S (sleeping)", "threads": 106}, "restart_count": 47, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.184, "sample_count": 1329, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:27:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.617, "last_frame_pushed_at": "2026-07-01T13:27:09+09:00", "last_frame_read_age_sec": 4.823, "last_frame_read_at": "2026-07-01T13:27:08+09:00", "mjpeg": {"age_sec": 4.597, "exists": true, "mtime": "2026-07-01T13:27:09.163010+09:00", "mtime_epoch": 1782880029.16301, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200969}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:27:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.477, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.978, "pushed": 3.978, "read": 3.945}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.305, "count": 118, "max_ms": 12.343, "p95_ms": 10.127}, "gstreamer_push": {"avg_ms": 0.859, "count": 119, "max_ms": 1.087, "p95_ms": 0.903}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.882, "count": 118, "max_ms": 48.698, "p95_ms": 40.89}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.999, "count": 118, "max_ms": 77.053, "p95_ms": 71.536}, "yolo11_640_fire_11class": {"avg_ms": 45.765, "count": 118, "max_ms": 66.862, "p95_ms": 55.223}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.555, "count": 118, "max_ms": 80.519, "p95_ms": 73.184}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.544, "count": 119, "max_ms": 236.497, "p95_ms": 223.727}, "read_interval": {"avg_ms": 214.467, "count": 118, "max_ms": 245.973, "p95_ms": 234.098}, "triton_wait": {"avg_ms": 60.779, "count": 118, "max_ms": 80.531, "p95_ms": 73.913}, "visualize": {"avg_ms": 9.735, "count": 118, "max_ms": 11.119, "p95_ms": 10.743}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 198}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5946.113, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f09451aa00] error while decoding MB 69 30, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:28:59+09:00", "last_error_summary": "[h264 @ 0x55f09451aa00] error while decoding MB 69 30, bytestream -11", "last_exit_at": "2026-07-01T13:27:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.458, "last_started_at": "2026-07-01T13:27:11+09:00", "pid": 10015, "process": {"exists": false, "pid": 10015}, "restart_count": 47, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.257, "sample_count": 1335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23765, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:28:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T13:28:59+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T13:28:59+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T13:28:59.176746+09:00", "mtime_epoch": 1782880139.176746, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183904}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:28:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.734, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.558, "pushed": 4.558, "read": 4.558}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.886, "count": 136, "max_ms": 13.333, "p95_ms": 11.362}, "gstreamer_push": {"avg_ms": 0.856, "count": 136, "max_ms": 1.0, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.09, "count": 136, "max_ms": 46.313, "p95_ms": 42.363}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.562, "count": 136, "max_ms": 80.649, "p95_ms": 75.218}, "yolo11_640_fire_11class": {"avg_ms": 47.494, "count": 136, "max_ms": 64.555, "p95_ms": 60.87}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.174, "count": 136, "max_ms": 81.399, "p95_ms": 75.876}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.244, "count": 136, "max_ms": 244.385, "p95_ms": 238.076}, "read_interval": {"avg_ms": 220.77, "count": 136, "max_ms": 254.321, "p95_ms": 247.722}, "triton_wait": {"avg_ms": 61.582, "count": 136, "max_ms": 81.411, "p95_ms": 78.03}, "visualize": {"avg_ms": 10.144, "count": 136, "max_ms": 13.291, "p95_ms": 11.644}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.835, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 198}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5945.316, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:28:59+09:00", "last_error_summary": "[h264 @ 0x55f09451aa00] error while decoding MB 69 30, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:28:59+09:00", "last_error_summary": "[h264 @ 0x55f09451aa00] error while decoding MB 69 30, bytestream -11", "last_exit_at": "2026-07-01T13:28:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.619, "last_started_at": "2026-07-01T13:27:11+09:00", "pid": 10015, "process": {"exists": false, "pid": 10015}, "restart_count": 47, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.257, "sample_count": 1335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23766, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:28:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:28:59+09:00", "last_frame_read_age_sec": 0.211, "last_frame_read_at": "2026-07-01T13:28:59+09:00", "mjpeg": {"age_sec": 0.201, "exists": true, "mtime": "2026-07-01T13:28:59.176746+09:00", "mtime_epoch": 1782880139.176746, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183904}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:28:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.734, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.567, "pushed": 4.567, "read": 4.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.868, "count": 136, "max_ms": 13.333, "p95_ms": 11.287}, "gstreamer_push": {"avg_ms": 0.855, "count": 137, "max_ms": 1.0, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.091, "count": 136, "max_ms": 46.313, "p95_ms": 42.363}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.579, "count": 136, "max_ms": 80.649, "p95_ms": 75.218}, "yolo11_640_fire_11class": {"avg_ms": 47.497, "count": 136, "max_ms": 64.555, "p95_ms": 60.87}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.18, "count": 136, "max_ms": 81.399, "p95_ms": 75.876}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.209, "count": 137, "max_ms": 244.385, "p95_ms": 238.076}, "read_interval": {"avg_ms": 220.77, "count": 136, "max_ms": 254.321, "p95_ms": 247.722}, "triton_wait": {"avg_ms": 61.588, "count": 136, "max_ms": 81.411, "p95_ms": 78.03}, "visualize": {"avg_ms": 10.131, "count": 136, "max_ms": 13.291, "p95_ms": 11.644}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 198}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5945.309, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:28:59+09:00", "last_error_summary": "[h264 @ 0x55f09451aa00] error while decoding MB 69 30, bytestream -11", "last_exit_at": "2026-07-01T13:28:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.622, "last_started_at": "2026-07-01T13:27:11+09:00", "pid": 10015, "process": {"exists": false, "pid": 10015}, "restart_count": 47, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.257, "sample_count": 1335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23766, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:28:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:28:59+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T13:28:59+09:00", "mjpeg": {"age_sec": 0.205, "exists": true, "mtime": "2026-07-01T13:28:59.176746+09:00", "mtime_epoch": 1782880139.176746, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183904}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:28:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.734, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.567, "pushed": 4.567, "read": 4.533}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.868, "count": 136, "max_ms": 13.333, "p95_ms": 11.287}, "gstreamer_push": {"avg_ms": 0.855, "count": 137, "max_ms": 1.0, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.091, "count": 136, "max_ms": 46.313, "p95_ms": 42.363}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.579, "count": 136, "max_ms": 80.649, "p95_ms": 75.218}, "yolo11_640_fire_11class": {"avg_ms": 47.497, "count": 136, "max_ms": 64.555, "p95_ms": 60.87}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.18, "count": 136, "max_ms": 81.399, "p95_ms": 75.876}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.209, "count": 137, "max_ms": 244.385, "p95_ms": 238.076}, "read_interval": {"avg_ms": 220.77, "count": 136, "max_ms": 254.321, "p95_ms": 247.722}, "triton_wait": {"avg_ms": 61.588, "count": 136, "max_ms": 81.411, "p95_ms": 78.03}, "visualize": {"avg_ms": 10.131, "count": 136, "max_ms": 13.291, "p95_ms": 11.644}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 198}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6053.91, "limit_mb": 8796093022207.996}, "detail": {"pid": 10122, "restart_count": 48, "start_count": 49}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:28:59+09:00", "last_error_summary": "[h264 @ 0x55f09451aa00] error while decoding MB 69 30, bytestream -11", "last_exit_at": "2026-07-01T13:29:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:29:01+09:00", "pid": 10122, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.566, "name": "ffmpeg", "pid": 10122, "rss_mb": 137.566, "state": "S (sleeping)", "threads": 106}, "restart_count": 48, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.096, "sample_count": 1315, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23766, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:28:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.607, "last_frame_pushed_at": "2026-07-01T13:28:59+09:00", "last_frame_read_age_sec": 4.815, "last_frame_read_at": "2026-07-01T13:28:59+09:00", "mjpeg": {"age_sec": 4.593, "exists": true, "mtime": "2026-07-01T13:28:59.388749+09:00", "mtime_epoch": 1782880139.3887491, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184047}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:29:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.734, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.868, "pushed": 3.868, "read": 3.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.794, "count": 115, "max_ms": 13.328, "p95_ms": 10.701}, "gstreamer_push": {"avg_ms": 0.853, "count": 116, "max_ms": 0.945, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.129, "count": 115, "max_ms": 46.313, "p95_ms": 42.363}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.989, "count": 115, "max_ms": 80.649, "p95_ms": 75.504}, "yolo11_640_fire_11class": {"avg_ms": 47.663, "count": 115, "max_ms": 64.555, "p95_ms": 60.87}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.103, "count": 115, "max_ms": 81.399, "p95_ms": 75.337}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.204, "count": 116, "max_ms": 242.761, "p95_ms": 237.276}, "read_interval": {"avg_ms": 220.718, "count": 115, "max_ms": 252.808, "p95_ms": 246.574}, "triton_wait": {"avg_ms": 61.582, "count": 115, "max_ms": 81.411, "p95_ms": 77.077}, "visualize": {"avg_ms": 10.167, "count": 115, "max_ms": 13.291, "p95_ms": 11.644}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 202}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5947.172, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:29:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.66, "last_started_at": "2026-07-01T13:29:01+09:00", "pid": 10122, "process": {"exists": false, "pid": 10122}, "restart_count": 48, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.251, "sample_count": 1323, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:31:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.049, "last_frame_pushed_at": "2026-07-01T13:31:08+09:00", "last_frame_read_age_sec": 0.041, "last_frame_read_at": "2026-07-01T13:31:08+09:00", "mjpeg": {"age_sec": 0.03, "exists": true, "mtime": "2026-07-01T13:31:08.610845+09:00", "mtime_epoch": 1782880268.6108446, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186519}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:31:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.992, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.578, "pushed": 4.578, "read": 4.578}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.802, "count": 136, "max_ms": 13.572, "p95_ms": 11.063}, "gstreamer_push": {"avg_ms": 0.855, "count": 137, "max_ms": 1.059, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.917, "count": 136, "max_ms": 46.942, "p95_ms": 39.829}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.708, "count": 136, "max_ms": 79.909, "p95_ms": 76.588}, "yolo11_640_fire_11class": {"avg_ms": 46.838, "count": 136, "max_ms": 65.373, "p95_ms": 61.193}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.627, "count": 136, "max_ms": 77.823, "p95_ms": 75.447}}, "truncated_count": 0}, "processing_total": {"avg_ms": 209.888, "count": 137, "max_ms": 241.599, "p95_ms": 235.512}, "read_interval": {"avg_ms": 219.664, "count": 137, "max_ms": 251.11, "p95_ms": 246.232}, "triton_wait": {"avg_ms": 61.203, "count": 136, "max_ms": 79.921, "p95_ms": 77.503}, "visualize": {"avg_ms": 10.065, "count": 136, "max_ms": 14.139, "p95_ms": 11.282}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 202}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5946.445, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:31:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.849, "last_started_at": "2026-07-01T13:29:01+09:00", "pid": 10122, "process": {"exists": false, "pid": 10122}, "restart_count": 48, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.251, "sample_count": 1323, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:31:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T13:31:08+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T13:31:08+09:00", "mjpeg": {"age_sec": 0.219, "exists": true, "mtime": "2026-07-01T13:31:08.610845+09:00", "mtime_epoch": 1782880268.6108446, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186519}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:31:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.992, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.584, "pushed": 4.584, "read": 4.551}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.808, "count": 136, "max_ms": 13.572, "p95_ms": 11.063}, "gstreamer_push": {"avg_ms": 0.855, "count": 137, "max_ms": 1.059, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.936, "count": 136, "max_ms": 46.942, "p95_ms": 39.829}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.575, "count": 136, "max_ms": 79.909, "p95_ms": 75.306}, "yolo11_640_fire_11class": {"avg_ms": 46.799, "count": 136, "max_ms": 65.373, "p95_ms": 61.193}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.511, "count": 136, "max_ms": 77.823, "p95_ms": 75.447}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.012, "count": 137, "max_ms": 241.599, "p95_ms": 235.512}, "read_interval": {"avg_ms": 219.684, "count": 136, "max_ms": 251.11, "p95_ms": 246.232}, "triton_wait": {"avg_ms": 61.071, "count": 136, "max_ms": 79.921, "p95_ms": 77.503}, "visualize": {"avg_ms": 10.075, "count": 136, "max_ms": 14.139, "p95_ms": 11.282}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 202}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5946.438, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:31:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.852, "last_started_at": "2026-07-01T13:29:01+09:00", "pid": 10122, "process": {"exists": false, "pid": 10122}, "restart_count": 48, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.251, "sample_count": 1323, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:31:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T13:31:08+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T13:31:08+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T13:31:08.610845+09:00", "mtime_epoch": 1782880268.6108446, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186519}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:31:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.992, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.584, "pushed": 4.584, "read": 4.55}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.808, "count": 136, "max_ms": 13.572, "p95_ms": 11.063}, "gstreamer_push": {"avg_ms": 0.855, "count": 137, "max_ms": 1.059, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.936, "count": 136, "max_ms": 46.942, "p95_ms": 39.829}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.575, "count": 136, "max_ms": 79.909, "p95_ms": 75.306}, "yolo11_640_fire_11class": {"avg_ms": 46.799, "count": 136, "max_ms": 65.373, "p95_ms": 61.193}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.511, "count": 136, "max_ms": 77.823, "p95_ms": 75.447}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.012, "count": 137, "max_ms": 241.599, "p95_ms": 235.512}, "read_interval": {"avg_ms": 219.684, "count": 136, "max_ms": 251.11, "p95_ms": 246.232}, "triton_wait": {"avg_ms": 61.071, "count": 136, "max_ms": 79.921, "p95_ms": 77.503}, "visualize": {"avg_ms": 10.075, "count": 136, "max_ms": 14.139, "p95_ms": 11.282}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 202}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6063.023, "limit_mb": 8796093022207.996}, "detail": {"pid": 10229, "restart_count": 49, "start_count": 50}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:31:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:31:11+09:00", "pid": 10229, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.426, "name": "ffmpeg", "pid": 10229, "rss_mb": 147.426, "state": "S (sleeping)", "threads": 106}, "restart_count": 49, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.268, "sample_count": 1301, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:31:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.725, "last_frame_pushed_at": "2026-07-01T13:31:08+09:00", "last_frame_read_age_sec": 4.951, "last_frame_read_at": "2026-07-01T13:31:08+09:00", "mjpeg": {"age_sec": 4.708, "exists": true, "mtime": "2026-07-01T13:31:08.842848+09:00", "mtime_epoch": 1782880268.8428483, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:31:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2361.992, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.848, "pushed": 3.848, "read": 3.814}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.859, "count": 114, "max_ms": 13.572, "p95_ms": 11.063}, "gstreamer_push": {"avg_ms": 0.855, "count": 115, "max_ms": 1.059, "p95_ms": 0.906}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.36, "count": 114, "max_ms": 46.942, "p95_ms": 40.474}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.168, "count": 114, "max_ms": 79.909, "p95_ms": 75.306}, "yolo11_640_fire_11class": {"avg_ms": 46.925, "count": 114, "max_ms": 65.373, "p95_ms": 61.426}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.547, "count": 114, "max_ms": 77.823, "p95_ms": 75.447}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.904, "count": 115, "max_ms": 241.599, "p95_ms": 237.618}, "read_interval": {"avg_ms": 220.563, "count": 114, "max_ms": 251.11, "p95_ms": 246.918}, "triton_wait": {"avg_ms": 60.974, "count": 114, "max_ms": 79.921, "p95_ms": 77.503}, "visualize": {"avg_ms": 10.138, "count": 114, "max_ms": 14.139, "p95_ms": 11.282}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 207}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5947.539, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:33:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.845, "last_started_at": "2026-07-01T13:31:11+09:00", "pid": 10229, "process": {"exists": false, "pid": 10229}, "restart_count": 49, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.007, "sample_count": 1339, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24925, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:33:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T13:33:19+09:00", "last_frame_read_age_sec": 0.206, "last_frame_read_at": "2026-07-01T13:33:19+09:00", "mjpeg": {"age_sec": 0.197, "exists": true, "mtime": "2026-07-01T13:33:19.197011+09:00", "mtime_epoch": 1782880399.1970105, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195807}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:33:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2362.508, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.727, "pushed": 4.727, "read": 4.693}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.29, "count": 141, "max_ms": 11.675, "p95_ms": 10.614}, "gstreamer_push": {"avg_ms": 0.85, "count": 141, "max_ms": 1.244, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.092, "count": 141, "max_ms": 50.268, "p95_ms": 42.469}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.405, "count": 141, "max_ms": 72.514, "p95_ms": 69.509}, "yolo11_640_fire_11class": {"avg_ms": 47.317, "count": 141, "max_ms": 61.891, "p95_ms": 57.508}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.428, "count": 141, "max_ms": 79.657, "p95_ms": 70.398}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.449, "count": 141, "max_ms": 231.362, "p95_ms": 214.76}, "read_interval": {"avg_ms": 213.034, "count": 140, "max_ms": 240.6, "p95_ms": 224.412}, "triton_wait": {"avg_ms": 60.548, "count": 141, "max_ms": 79.669, "p95_ms": 71.203}, "visualize": {"avg_ms": 9.685, "count": 141, "max_ms": 11.186, "p95_ms": 10.484}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.829, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 207}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5947.531, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:33:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.848, "last_started_at": "2026-07-01T13:31:11+09:00", "pid": 10229, "process": {"exists": false, "pid": 10229}, "restart_count": 49, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.007, "sample_count": 1339, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24925, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:33:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T13:33:19+09:00", "last_frame_read_age_sec": 0.209, "last_frame_read_at": "2026-07-01T13:33:19+09:00", "mjpeg": {"age_sec": 0.201, "exists": true, "mtime": "2026-07-01T13:33:19.197011+09:00", "mtime_epoch": 1782880399.1970105, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195807}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:33:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2362.508, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.726, "pushed": 4.726, "read": 4.693}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.29, "count": 141, "max_ms": 11.675, "p95_ms": 10.614}, "gstreamer_push": {"avg_ms": 0.85, "count": 141, "max_ms": 1.244, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.092, "count": 141, "max_ms": 50.268, "p95_ms": 42.469}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.405, "count": 141, "max_ms": 72.514, "p95_ms": 69.509}, "yolo11_640_fire_11class": {"avg_ms": 47.317, "count": 141, "max_ms": 61.891, "p95_ms": 57.508}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.428, "count": 141, "max_ms": 79.657, "p95_ms": 70.398}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.449, "count": 141, "max_ms": 231.362, "p95_ms": 214.76}, "read_interval": {"avg_ms": 213.034, "count": 140, "max_ms": 240.6, "p95_ms": 224.412}, "triton_wait": {"avg_ms": 60.548, "count": 141, "max_ms": 79.669, "p95_ms": 71.203}, "visualize": {"avg_ms": 9.685, "count": 141, "max_ms": 11.186, "p95_ms": 10.484}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.832, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 207}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6053.539, "limit_mb": 8796093022207.996}, "detail": {"pid": 10336, "restart_count": 50, "start_count": 51}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:33:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:33:22+09:00", "pid": 10336, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.57, "name": "ffmpeg", "pid": 10336, "rss_mb": 136.57, "state": "S (sleeping)", "threads": 106}, "restart_count": 50, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.216, "sample_count": 1319, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24925, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:33:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.777, "last_frame_pushed_at": "2026-07-01T13:33:19+09:00", "last_frame_read_age_sec": 4.978, "last_frame_read_at": "2026-07-01T13:33:19+09:00", "mjpeg": {"age_sec": 4.762, "exists": true, "mtime": "2026-07-01T13:33:19.405014+09:00", "mtime_epoch": 1782880399.405014, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195804}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:33:24+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2362.508, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.979, "pushed": 3.979, "read": 3.946}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.287, "count": 118, "max_ms": 11.419, "p95_ms": 10.718}, "gstreamer_push": {"avg_ms": 0.85, "count": 119, "max_ms": 1.244, "p95_ms": 0.892}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.294, "count": 118, "max_ms": 50.268, "p95_ms": 43.13}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.64, "count": 118, "max_ms": 72.514, "p95_ms": 70.986}, "yolo11_640_fire_11class": {"avg_ms": 47.211, "count": 118, "max_ms": 61.891, "p95_ms": 59.435}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.379, "count": 118, "max_ms": 79.657, "p95_ms": 70.847}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.373, "count": 119, "max_ms": 231.362, "p95_ms": 216.557}, "read_interval": {"avg_ms": 212.935, "count": 118, "max_ms": 240.6, "p95_ms": 226.111}, "triton_wait": {"avg_ms": 60.52, "count": 118, "max_ms": 79.669, "p95_ms": 71.358}, "visualize": {"avg_ms": 9.692, "count": 118, "max_ms": 11.186, "p95_ms": 10.484}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 212}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5948.281, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:35:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.27, "last_started_at": "2026-07-01T13:33:22+09:00", "pid": 10336, "process": {"exists": false, "pid": 10336}, "restart_count": 50, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.034, "sample_count": 1359, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25520, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:35:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T13:35:30+09:00", "last_frame_read_age_sec": 0.206, "last_frame_read_at": "2026-07-01T13:35:30+09:00", "mjpeg": {"age_sec": 0.196, "exists": true, "mtime": "2026-07-01T13:35:30.239222+09:00", "mtime_epoch": 1782880530.239222, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189905}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:35:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2362.766, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.702, "pushed": 4.702, "read": 4.669}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.24, "count": 140, "max_ms": 12.593, "p95_ms": 10.003}, "gstreamer_push": {"avg_ms": 0.858, "count": 141, "max_ms": 1.151, "p95_ms": 0.903}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.717, "count": 140, "max_ms": 48.185, "p95_ms": 40.709}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.714, "count": 140, "max_ms": 76.575, "p95_ms": 69.705}, "yolo11_640_fire_11class": {"avg_ms": 47.997, "count": 140, "max_ms": 62.603, "p95_ms": 58.269}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.947, "count": 140, "max_ms": 79.694, "p95_ms": 72.882}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.291, "count": 141, "max_ms": 234.053, "p95_ms": 217.184}, "read_interval": {"avg_ms": 214.084, "count": 140, "max_ms": 244.697, "p95_ms": 226.507}, "triton_wait": {"avg_ms": 61.216, "count": 140, "max_ms": 79.707, "p95_ms": 73.797}, "visualize": {"avg_ms": 9.929, "count": 140, "max_ms": 15.176, "p95_ms": 11.227}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 212}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5948.273, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:35:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.273, "last_started_at": "2026-07-01T13:33:22+09:00", "pid": 10336, "process": {"exists": false, "pid": 10336}, "restart_count": 50, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.034, "sample_count": 1359, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25520, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:35:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T13:35:30+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T13:35:30+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T13:35:30.239222+09:00", "mtime_epoch": 1782880530.239222, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189905}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:35:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2362.766, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.702, "pushed": 4.702, "read": 4.669}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.24, "count": 140, "max_ms": 12.593, "p95_ms": 10.003}, "gstreamer_push": {"avg_ms": 0.858, "count": 141, "max_ms": 1.151, "p95_ms": 0.903}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.717, "count": 140, "max_ms": 48.185, "p95_ms": 40.709}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.714, "count": 140, "max_ms": 76.575, "p95_ms": 69.705}, "yolo11_640_fire_11class": {"avg_ms": 47.997, "count": 140, "max_ms": 62.603, "p95_ms": 58.269}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.947, "count": 140, "max_ms": 79.694, "p95_ms": 72.882}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.291, "count": 141, "max_ms": 234.053, "p95_ms": 217.184}, "read_interval": {"avg_ms": 214.084, "count": 140, "max_ms": 244.697, "p95_ms": 226.507}, "triton_wait": {"avg_ms": 61.216, "count": 140, "max_ms": 79.707, "p95_ms": 73.797}, "visualize": {"avg_ms": 9.929, "count": 140, "max_ms": 15.176, "p95_ms": 11.227}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 212}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6056.422, "limit_mb": 8796093022207.996}, "detail": {"pid": 10443, "restart_count": 51, "start_count": 52}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:35:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:35:33+09:00", "pid": 10443, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.43, "name": "ffmpeg", "pid": 10443, "rss_mb": 137.43, "state": "S (sleeping)", "threads": 106}, "restart_count": 51, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.034, "sample_count": 1337, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25520, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:35:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.828, "last_frame_pushed_at": "2026-07-01T13:35:30+09:00", "last_frame_read_age_sec": 5.029, "last_frame_read_at": "2026-07-01T13:35:30+09:00", "mjpeg": {"age_sec": 4.811, "exists": true, "mtime": "2026-07-01T13:35:30.447226+09:00", "mtime_epoch": 1782880530.4472256, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190732}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:35:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2362.766, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.952, "pushed": 3.952, "read": 3.918}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.244, "count": 118, "max_ms": 12.593, "p95_ms": 10.17}, "gstreamer_push": {"avg_ms": 0.859, "count": 118, "max_ms": 1.151, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.465, "count": 118, "max_ms": 48.185, "p95_ms": 40.709}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.922, "count": 118, "max_ms": 76.575, "p95_ms": 69.158}, "yolo11_640_fire_11class": {"avg_ms": 47.626, "count": 118, "max_ms": 62.603, "p95_ms": 58.845}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.509, "count": 118, "max_ms": 79.694, "p95_ms": 72.339}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.175, "count": 118, "max_ms": 234.053, "p95_ms": 218.242}, "read_interval": {"avg_ms": 213.965, "count": 117, "max_ms": 244.697, "p95_ms": 227.506}, "triton_wait": {"avg_ms": 60.762, "count": 118, "max_ms": 79.707, "p95_ms": 73.755}, "visualize": {"avg_ms": 9.967, "count": 118, "max_ms": 15.176, "p95_ms": 11.397}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.86, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5949.586, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:37:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.347, "last_started_at": "2026-07-01T13:35:33+09:00", "pid": 10443, "process": {"exists": false, "pid": 10443}, "restart_count": 51, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.032, "sample_count": 1366, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26113, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:37:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:37:41+09:00", "last_frame_read_age_sec": 0.239, "last_frame_read_at": "2026-07-01T13:37:41+09:00", "mjpeg": {"age_sec": 0.231, "exists": true, "mtime": "2026-07-01T13:37:41.373465+09:00", "mtime_epoch": 1782880661.3734646, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194384}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:37:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.023, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.688, "pushed": 4.688, "read": 4.654}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.358, "count": 140, "max_ms": 13.993, "p95_ms": 11.446}, "gstreamer_push": {"avg_ms": 0.866, "count": 140, "max_ms": 1.59, "p95_ms": 0.918}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.297, "count": 140, "max_ms": 48.218, "p95_ms": 41.033}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.408, "count": 140, "max_ms": 75.883, "p95_ms": 72.883}, "yolo11_640_fire_11class": {"avg_ms": 47.997, "count": 140, "max_ms": 63.282, "p95_ms": 60.07}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.548, "count": 140, "max_ms": 75.804, "p95_ms": 73.562}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.935, "count": 140, "max_ms": 239.51, "p95_ms": 218.37}, "read_interval": {"avg_ms": 214.623, "count": 139, "max_ms": 252.073, "p95_ms": 227.869}, "triton_wait": {"avg_ms": 61.812, "count": 140, "max_ms": 75.895, "p95_ms": 73.898}, "visualize": {"avg_ms": 9.833, "count": 140, "max_ms": 13.787, "p95_ms": 10.745}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.864, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5949.578, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:37:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.351, "last_started_at": "2026-07-01T13:35:33+09:00", "pid": 10443, "process": {"exists": false, "pid": 10443}, "restart_count": 51, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.032, "sample_count": 1366, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26113, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:37:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:37:41+09:00", "last_frame_read_age_sec": 0.242, "last_frame_read_at": "2026-07-01T13:37:41+09:00", "mjpeg": {"age_sec": 0.235, "exists": true, "mtime": "2026-07-01T13:37:41.373465+09:00", "mtime_epoch": 1782880661.3734646, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194384}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:37:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.023, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.687, "pushed": 4.687, "read": 4.654}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.358, "count": 140, "max_ms": 13.993, "p95_ms": 11.446}, "gstreamer_push": {"avg_ms": 0.866, "count": 140, "max_ms": 1.59, "p95_ms": 0.918}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.297, "count": 140, "max_ms": 48.218, "p95_ms": 41.033}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.408, "count": 140, "max_ms": 75.883, "p95_ms": 72.883}, "yolo11_640_fire_11class": {"avg_ms": 47.997, "count": 140, "max_ms": 63.282, "p95_ms": 60.07}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.548, "count": 140, "max_ms": 75.804, "p95_ms": 73.562}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.935, "count": 140, "max_ms": 239.51, "p95_ms": 218.37}, "read_interval": {"avg_ms": 214.623, "count": 139, "max_ms": 252.073, "p95_ms": 227.869}, "triton_wait": {"avg_ms": 61.812, "count": 140, "max_ms": 75.895, "p95_ms": 73.898}, "visualize": {"avg_ms": 9.833, "count": 140, "max_ms": 13.787, "p95_ms": 10.745}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5960.781, "limit_mb": 8796093022207.996}, "detail": {"pid": 10550, "restart_count": 52, "start_count": 53}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:37:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:37:44+09:00", "pid": 10550, "process": {"exists": true, "fd_count": 4, "hwm_mb": 45.887, "name": "ffmpeg", "pid": 10550, "rss_mb": 45.887, "state": "S (sleeping)", "threads": 1}, "restart_count": 52, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.024, "sample_count": 1344, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26113, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:37:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.66, "last_frame_pushed_at": "2026-07-01T13:37:41+09:00", "last_frame_read_age_sec": 4.896, "last_frame_read_at": "2026-07-01T13:37:41+09:00", "mjpeg": {"age_sec": 4.644, "exists": true, "mtime": "2026-07-01T13:37:41.617469+09:00", "mtime_epoch": 1782880661.6174688, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195199}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:37:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.023, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.97, "pushed": 3.97, "read": 3.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.263, "count": 118, "max_ms": 13.553, "p95_ms": 10.534}, "gstreamer_push": {"avg_ms": 0.869, "count": 119, "max_ms": 1.59, "p95_ms": 0.963}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.105, "count": 118, "max_ms": 48.218, "p95_ms": 41.357}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.052, "count": 118, "max_ms": 75.883, "p95_ms": 72.883}, "yolo11_640_fire_11class": {"avg_ms": 47.68, "count": 118, "max_ms": 63.282, "p95_ms": 61.103}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.185, "count": 118, "max_ms": 75.804, "p95_ms": 74.41}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.743, "count": 119, "max_ms": 239.51, "p95_ms": 221.264}, "read_interval": {"avg_ms": 214.367, "count": 118, "max_ms": 252.073, "p95_ms": 230.533}, "triton_wait": {"avg_ms": 61.45, "count": 118, "max_ms": 75.895, "p95_ms": 74.644}, "visualize": {"avg_ms": 9.804, "count": 118, "max_ms": 13.771, "p95_ms": 10.69}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 222}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5950.496, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:39:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.238, "last_started_at": "2026-07-01T13:37:44+09:00", "pid": 10550, "process": {"exists": false, "pid": 10550}, "restart_count": 52, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.718, "sample_count": 1279, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26654, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:39:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T13:39:59+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T13:39:59+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-01T13:39:59.295848+09:00", "mtime_epoch": 1782880799.2958477, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200020}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:39:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.281, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.709, "pushed": 4.709, "read": 4.676}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.157, "count": 140, "max_ms": 12.752, "p95_ms": 9.939}, "gstreamer_push": {"avg_ms": 0.852, "count": 141, "max_ms": 0.928, "p95_ms": 0.885}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.921, "count": 140, "max_ms": 47.763, "p95_ms": 41.679}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.846, "count": 140, "max_ms": 79.802, "p95_ms": 74.296}, "yolo11_640_fire_11class": {"avg_ms": 46.481, "count": 140, "max_ms": 65.109, "p95_ms": 58.176}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.788, "count": 140, "max_ms": 79.521, "p95_ms": 75.041}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.926, "count": 141, "max_ms": 229.066, "p95_ms": 218.163}, "read_interval": {"avg_ms": 213.795, "count": 140, "max_ms": 238.368, "p95_ms": 228.324}, "triton_wait": {"avg_ms": 61.787, "count": 140, "max_ms": 79.814, "p95_ms": 76.141}, "visualize": {"avg_ms": 9.712, "count": 140, "max_ms": 12.799, "p95_ms": 10.422}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 222}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5950.488, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:39:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.242, "last_started_at": "2026-07-01T13:37:44+09:00", "pid": 10550, "process": {"exists": false, "pid": 10550}, "restart_count": 52, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.718, "sample_count": 1279, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26654, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:39:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T13:39:59+09:00", "last_frame_read_age_sec": 0.217, "last_frame_read_at": "2026-07-01T13:39:59+09:00", "mjpeg": {"age_sec": 0.207, "exists": true, "mtime": "2026-07-01T13:39:59.295848+09:00", "mtime_epoch": 1782880799.2958477, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200020}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:39:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.281, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.709, "pushed": 4.709, "read": 4.675}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.157, "count": 140, "max_ms": 12.752, "p95_ms": 9.939}, "gstreamer_push": {"avg_ms": 0.852, "count": 141, "max_ms": 0.928, "p95_ms": 0.885}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.921, "count": 140, "max_ms": 47.763, "p95_ms": 41.679}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.846, "count": 140, "max_ms": 79.802, "p95_ms": 74.296}, "yolo11_640_fire_11class": {"avg_ms": 46.481, "count": 140, "max_ms": 65.109, "p95_ms": 58.176}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.788, "count": 140, "max_ms": 79.521, "p95_ms": 75.041}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.926, "count": 141, "max_ms": 229.066, "p95_ms": 218.163}, "read_interval": {"avg_ms": 213.795, "count": 140, "max_ms": 238.368, "p95_ms": 228.324}, "triton_wait": {"avg_ms": 61.787, "count": 140, "max_ms": 79.814, "p95_ms": 76.141}, "visualize": {"avg_ms": 9.712, "count": 140, "max_ms": 12.799, "p95_ms": 10.422}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 222}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6058.617, "limit_mb": 8796093022207.996}, "detail": {"pid": 10657, "restart_count": 53, "start_count": 54}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:31:08+09:00", "last_error_summary": "[h264 @ 0x561fe9fc8c00] error while decoding MB 11 26, bytestream -13", "last_exit_at": "2026-07-01T13:40:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:40:01+09:00", "pid": 10657, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.578, "name": "ffmpeg", "pid": 10657, "rss_mb": 138.578, "state": "S (sleeping)", "threads": 106}, "restart_count": 53, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.911, "sample_count": 1258, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26654, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:39:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.43, "last_frame_pushed_at": "2026-07-01T13:39:59+09:00", "last_frame_read_age_sec": 4.639, "last_frame_read_at": "2026-07-01T13:39:59+09:00", "mjpeg": {"age_sec": 4.412, "exists": true, "mtime": "2026-07-01T13:39:59.511852+09:00", "mtime_epoch": 1782880799.5118515, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200538}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:40:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.281, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.012, "pushed": 4.012, "read": 3.979}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.156, "count": 119, "max_ms": 12.752, "p95_ms": 9.939}, "gstreamer_push": {"avg_ms": 0.854, "count": 120, "max_ms": 0.928, "p95_ms": 0.885}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.922, "count": 119, "max_ms": 47.728, "p95_ms": 41.679}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.867, "count": 119, "max_ms": 79.802, "p95_ms": 74.296}, "yolo11_640_fire_11class": {"avg_ms": 46.534, "count": 119, "max_ms": 65.109, "p95_ms": 58.847}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.679, "count": 119, "max_ms": 79.521, "p95_ms": 75.041}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.195, "count": 120, "max_ms": 229.066, "p95_ms": 218.182}, "read_interval": {"avg_ms": 214.082, "count": 119, "max_ms": 238.368, "p95_ms": 229.997}, "triton_wait": {"avg_ms": 61.744, "count": 119, "max_ms": 79.814, "p95_ms": 76.396}, "visualize": {"avg_ms": 9.744, "count": 119, "max_ms": 12.799, "p95_ms": 10.927}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5952.832, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557c53f82080] error while decoding MB 66 19, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:41:49+09:00", "last_error_summary": "[h264 @ 0x557c53f82080] error while decoding MB 66 19, bytestream -5", "last_exit_at": "2026-07-01T13:40:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.542, "last_started_at": "2026-07-01T13:40:01+09:00", "pid": 10657, "process": {"exists": false, "pid": 10657}, "restart_count": 53, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.71, "sample_count": 1256, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27124, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:41:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T13:41:49+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T13:41:49+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T13:41:49.421765+09:00", "mtime_epoch": 1782880909.4217649, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199659}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:41:49+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.539, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.486, "pushed": 4.486, "read": 4.486}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk"], "latency_ms": {"analyze": {"avg_ms": 9.755, "count": 133, "max_ms": 14.389, "p95_ms": 11.575}, "gstreamer_push": {"avg_ms": 0.843, "count": 134, "max_ms": 1.031, "p95_ms": 0.893}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.26, "count": 133, "max_ms": 47.836, "p95_ms": 41.847}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.218, "count": 133, "max_ms": 80.187, "p95_ms": 68.354}, "yolo11_640_fire_11class": {"avg_ms": 42.896, "count": 133, "max_ms": 58.734, "p95_ms": 51.157}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.08, "count": 133, "max_ms": 78.75, "p95_ms": 69.859}}, "truncated_count": 0}, "processing_total": {"avg_ms": 214.61, "count": 134, "max_ms": 248.295, "p95_ms": 231.777}, "read_interval": {"avg_ms": 224.215, "count": 134, "max_ms": 257.948, "p95_ms": 241.286}, "triton_wait": {"avg_ms": 57.423, "count": 133, "max_ms": 80.199, "p95_ms": 69.871}, "visualize": {"avg_ms": 10.368, "count": 133, "max_ms": 12.107, "p95_ms": 11.268}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.872, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5952.406, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:41:49+09:00", "last_error_summary": "[h264 @ 0x557c53f82080] error while decoding MB 66 19, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:41:49+09:00", "last_error_summary": "[h264 @ 0x557c53f82080] error while decoding MB 66 19, bytestream -5", "last_exit_at": "2026-07-01T13:41:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.718, "last_started_at": "2026-07-01T13:40:01+09:00", "pid": 10657, "process": {"exists": false, "pid": 10657}, "restart_count": 53, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.71, "sample_count": 1256, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27125, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:41:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:41:49+09:00", "last_frame_read_age_sec": 0.228, "last_frame_read_at": "2026-07-01T13:41:49+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-01T13:41:49.421765+09:00", "mtime_epoch": 1782880909.4217649, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199659}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:41:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.539, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.495, "pushed": 4.495, "read": 4.461}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk"], "latency_ms": {"analyze": {"avg_ms": 9.771, "count": 134, "max_ms": 14.389, "p95_ms": 11.757}, "gstreamer_push": {"avg_ms": 0.843, "count": 134, "max_ms": 1.031, "p95_ms": 0.893}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.247, "count": 134, "max_ms": 47.836, "p95_ms": 41.847}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.167, "count": 134, "max_ms": 80.187, "p95_ms": 68.354}, "yolo11_640_fire_11class": {"avg_ms": 42.896, "count": 134, "max_ms": 58.734, "p95_ms": 51.157}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.072, "count": 134, "max_ms": 78.75, "p95_ms": 69.859}}, "truncated_count": 0}, "processing_total": {"avg_ms": 214.598, "count": 134, "max_ms": 248.295, "p95_ms": 231.777}, "read_interval": {"avg_ms": 224.117, "count": 133, "max_ms": 257.948, "p95_ms": 241.286}, "triton_wait": {"avg_ms": 57.412, "count": 134, "max_ms": 80.199, "p95_ms": 69.871}, "visualize": {"avg_ms": 10.386, "count": 134, "max_ms": 12.709, "p95_ms": 11.311}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.812, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5952.398, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:41:49+09:00", "last_error_summary": "[h264 @ 0x557c53f82080] error while decoding MB 66 19, bytestream -5", "last_exit_at": "2026-07-01T13:41:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.722, "last_started_at": "2026-07-01T13:40:01+09:00", "pid": 10657, "process": {"exists": false, "pid": 10657}, "restart_count": 53, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.71, "sample_count": 1256, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27125, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:41:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T13:41:49+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T13:41:49+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T13:41:49.421765+09:00", "mtime_epoch": 1782880909.4217649, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199659}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:41:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.539, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.494, "pushed": 4.494, "read": 4.461}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk"], "latency_ms": {"analyze": {"avg_ms": 9.771, "count": 134, "max_ms": 14.389, "p95_ms": 11.757}, "gstreamer_push": {"avg_ms": 0.843, "count": 134, "max_ms": 1.031, "p95_ms": 0.893}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.247, "count": 134, "max_ms": 47.836, "p95_ms": 41.847}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.167, "count": 134, "max_ms": 80.187, "p95_ms": 68.354}, "yolo11_640_fire_11class": {"avg_ms": 42.896, "count": 134, "max_ms": 58.734, "p95_ms": 51.157}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.072, "count": 134, "max_ms": 78.75, "p95_ms": 69.859}}, "truncated_count": 0}, "processing_total": {"avg_ms": 214.598, "count": 134, "max_ms": 248.295, "p95_ms": 231.777}, "read_interval": {"avg_ms": 224.117, "count": 133, "max_ms": 257.948, "p95_ms": 241.286}, "triton_wait": {"avg_ms": 57.412, "count": 134, "max_ms": 80.199, "p95_ms": 69.871}, "visualize": {"avg_ms": 10.386, "count": 134, "max_ms": 12.709, "p95_ms": 11.311}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.815, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6060.586, "limit_mb": 8796093022207.996}, "detail": {"pid": 10764, "restart_count": 54, "start_count": 55}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:41:49+09:00", "last_error_summary": "[h264 @ 0x557c53f82080] error while decoding MB 66 19, bytestream -5", "last_exit_at": "2026-07-01T13:41:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:41:52+09:00", "pid": 10764, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.68, "name": "ffmpeg", "pid": 10764, "rss_mb": 138.68, "state": "S (sleeping)", "threads": 106}, "restart_count": 54, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.79, "sample_count": 1234, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27125, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:41:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.753, "last_frame_pushed_at": "2026-07-01T13:41:49+09:00", "last_frame_read_age_sec": 4.979, "last_frame_read_at": "2026-07-01T13:41:49+09:00", "mjpeg": {"age_sec": 4.738, "exists": true, "mtime": "2026-07-01T13:41:49.653769+09:00", "mtime_epoch": 1782880909.6537688, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200599}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:41:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.539, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.795, "pushed": 3.795, "read": 3.761}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk"], "latency_ms": {"analyze": {"avg_ms": 9.694, "count": 113, "max_ms": 14.389, "p95_ms": 11.477}, "gstreamer_push": {"avg_ms": 0.842, "count": 113, "max_ms": 1.031, "p95_ms": 0.893}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.06, "count": 113, "max_ms": 47.836, "p95_ms": 41.324}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.477, "count": 113, "max_ms": 80.187, "p95_ms": 65.842}, "yolo11_640_fire_11class": {"avg_ms": 42.31, "count": 113, "max_ms": 56.294, "p95_ms": 48.956}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.434, "count": 113, "max_ms": 78.75, "p95_ms": 69.556}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.924, "count": 113, "max_ms": 248.295, "p95_ms": 231.777}, "read_interval": {"avg_ms": 223.407, "count": 112, "max_ms": 257.948, "p95_ms": 241.286}, "triton_wait": {"avg_ms": 56.771, "count": 113, "max_ms": 80.199, "p95_ms": 69.568}, "visualize": {"avg_ms": 10.39, "count": 113, "max_ms": 12.709, "p95_ms": 11.33}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.777, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 230}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.777, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557a6060f740] error while decoding MB 7 24, bytestream -7 [h264 @ 0x557a610fca80] error while decoding MB 52 5, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:43:59+09:00", "last_error_summary": "[h264 @ 0x557a6060f740] error while decoding MB 7 24, bytestream -7 [h264 @ 0x557a610fca80] error while decoding MB 52 5, bytestream -5", "last_exit_at": "2026-07-01T13:41:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.148, "last_started_at": "2026-07-01T13:41:52+09:00", "pid": 10764, "process": {"exists": false, "pid": 10764}, "restart_count": 54, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.478, "sample_count": 1321, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27695, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:43:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T13:43:59+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T13:43:59+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T13:43:59.496042+09:00", "mtime_epoch": 1782881039.4960415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191990}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:43:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.797, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.597, "pushed": 4.597, "read": 4.597}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.61, "count": 137, "max_ms": 14.937, "p95_ms": 11.489}, "gstreamer_push": {"avg_ms": 0.864, "count": 137, "max_ms": 1.253, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.966, "count": 137, "max_ms": 48.522, "p95_ms": 44.392}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.672, "count": 137, "max_ms": 79.991, "p95_ms": 73.804}, "yolo11_640_fire_11class": {"avg_ms": 49.644, "count": 137, "max_ms": 64.919, "p95_ms": 61.112}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.763, "count": 137, "max_ms": 81.431, "p95_ms": 74.725}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.954, "count": 137, "max_ms": 241.146, "p95_ms": 227.275}, "read_interval": {"avg_ms": 218.682, "count": 137, "max_ms": 250.555, "p95_ms": 236.483}, "triton_wait": {"avg_ms": 63.203, "count": 137, "max_ms": 81.443, "p95_ms": 75.686}, "visualize": {"avg_ms": 9.826, "count": 137, "max_ms": 13.816, "p95_ms": 10.992}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.805, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 230}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.211, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:43:59+09:00", "last_error_summary": "[h264 @ 0x557a6060f740] error while decoding MB 7 24, bytestream -7 [h264 @ 0x557a610fca80] error while decoding MB 52 5, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:43:59+09:00", "last_error_summary": "[h264 @ 0x557a6060f740] error while decoding MB 7 24, bytestream -7 [h264 @ 0x557a610fca80] error while decoding MB 52 5, bytestream -5", "last_exit_at": "2026-07-01T13:43:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.32, "last_started_at": "2026-07-01T13:41:52+09:00", "pid": 10764, "process": {"exists": false, "pid": 10764}, "restart_count": 54, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.478, "sample_count": 1321, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:43:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T13:43:59+09:00", "last_frame_read_age_sec": 0.223, "last_frame_read_at": "2026-07-01T13:43:59+09:00", "mjpeg": {"age_sec": 0.214, "exists": true, "mtime": "2026-07-01T13:43:59.496042+09:00", "mtime_epoch": 1782881039.4960415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191990}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:43:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.797, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.604, "pushed": 4.604, "read": 4.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.618, "count": 137, "max_ms": 14.937, "p95_ms": 11.489}, "gstreamer_push": {"avg_ms": 0.864, "count": 138, "max_ms": 1.253, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.947, "count": 137, "max_ms": 48.522, "p95_ms": 44.392}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.766, "count": 137, "max_ms": 79.991, "p95_ms": 73.804}, "yolo11_640_fire_11class": {"avg_ms": 49.696, "count": 137, "max_ms": 64.919, "p95_ms": 61.112}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.817, "count": 137, "max_ms": 81.431, "p95_ms": 74.725}}, "truncated_count": 0}, "processing_total": {"avg_ms": 209.008, "count": 138, "max_ms": 241.146, "p95_ms": 227.275}, "read_interval": {"avg_ms": 218.682, "count": 137, "max_ms": 250.555, "p95_ms": 236.483}, "triton_wait": {"avg_ms": 63.285, "count": 137, "max_ms": 81.443, "p95_ms": 75.686}, "visualize": {"avg_ms": 9.796, "count": 137, "max_ms": 12.449, "p95_ms": 10.952}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 230}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.203, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:43:59+09:00", "last_error_summary": "[h264 @ 0x557a6060f740] error while decoding MB 7 24, bytestream -7 [h264 @ 0x557a610fca80] error while decoding MB 52 5, bytestream -5", "last_exit_at": "2026-07-01T13:43:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.323, "last_started_at": "2026-07-01T13:41:52+09:00", "pid": 10764, "process": {"exists": false, "pid": 10764}, "restart_count": 54, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.478, "sample_count": 1321, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:43:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T13:43:59+09:00", "last_frame_read_age_sec": 0.226, "last_frame_read_at": "2026-07-01T13:43:59+09:00", "mjpeg": {"age_sec": 0.218, "exists": true, "mtime": "2026-07-01T13:43:59.496042+09:00", "mtime_epoch": 1782881039.4960415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191990}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:43:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.797, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.603, "pushed": 4.603, "read": 4.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.618, "count": 137, "max_ms": 14.937, "p95_ms": 11.489}, "gstreamer_push": {"avg_ms": 0.864, "count": 138, "max_ms": 1.253, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.947, "count": 137, "max_ms": 48.522, "p95_ms": 44.392}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.766, "count": 137, "max_ms": 79.991, "p95_ms": 73.804}, "yolo11_640_fire_11class": {"avg_ms": 49.696, "count": 137, "max_ms": 64.919, "p95_ms": 61.112}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.817, "count": 137, "max_ms": 81.431, "p95_ms": 74.725}}, "truncated_count": 0}, "processing_total": {"avg_ms": 209.008, "count": 138, "max_ms": 241.146, "p95_ms": 227.275}, "read_interval": {"avg_ms": 218.682, "count": 137, "max_ms": 250.555, "p95_ms": 236.483}, "triton_wait": {"avg_ms": 63.285, "count": 137, "max_ms": 81.443, "p95_ms": 75.686}, "visualize": {"avg_ms": 9.796, "count": 137, "max_ms": 12.449, "p95_ms": 10.952}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 230}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6060.746, "limit_mb": 8796093022207.996}, "detail": {"pid": 10871, "restart_count": 55, "start_count": 56}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:43:59+09:00", "last_error_summary": "[h264 @ 0x557a6060f740] error while decoding MB 7 24, bytestream -7 [h264 @ 0x557a610fca80] error while decoding MB 52 5, bytestream -5", "last_exit_at": "2026-07-01T13:44:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:44:02+09:00", "pid": 10871, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.262, "name": "ffmpeg", "pid": 10871, "rss_mb": 139.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 55, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.55, "sample_count": 1298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:43:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.947, "last_frame_pushed_at": "2026-07-01T13:43:59+09:00", "last_frame_read_age_sec": 5.164, "last_frame_read_at": "2026-07-01T13:43:59+09:00", "mjpeg": {"age_sec": 4.932, "exists": true, "mtime": "2026-07-01T13:43:59.720046+09:00", "mtime_epoch": 1782881039.7200456, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193824}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:44:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2363.797, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.853, "pushed": 3.853, "read": 3.82}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.665, "count": 115, "max_ms": 14.937, "p95_ms": 11.489}, "gstreamer_push": {"avg_ms": 0.864, "count": 115, "max_ms": 1.253, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.013, "count": 115, "max_ms": 48.522, "p95_ms": 44.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.495, "count": 115, "max_ms": 79.356, "p95_ms": 73.424}, "yolo11_640_fire_11class": {"avg_ms": 49.518, "count": 115, "max_ms": 62.774, "p95_ms": 60.014}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.744, "count": 115, "max_ms": 81.431, "p95_ms": 74.725}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.624, "count": 115, "max_ms": 241.146, "p95_ms": 233.231}, "read_interval": {"avg_ms": 218.28, "count": 114, "max_ms": 250.555, "p95_ms": 242.912}, "triton_wait": {"avg_ms": 63.103, "count": 115, "max_ms": 81.443, "p95_ms": 75.458}, "visualize": {"avg_ms": 9.789, "count": 115, "max_ms": 12.449, "p95_ms": 10.883}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 235}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5955.07, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55937d706ec0] error while decoding MB 78 32, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:46:09+09:00", "last_error_summary": "[h264 @ 0x55937d706ec0] error while decoding MB 78 32, bytestream -15", "last_exit_at": "2026-07-01T13:44:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.699, "last_started_at": "2026-07-01T13:44:02+09:00", "pid": 10871, "process": {"exists": false, "pid": 10871}, "restart_count": 55, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.989, "sample_count": 1330, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28276, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:46:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-01T13:46:09+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-01T13:46:09+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-01T13:46:09.298324+09:00", "mtime_epoch": 1782881169.2983243, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:46:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.055, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.657, "pushed": 4.657, "read": 4.657}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.722, "count": 139, "max_ms": 14.863, "p95_ms": 12.458}, "gstreamer_push": {"avg_ms": 0.877, "count": 139, "max_ms": 1.053, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.963, "count": 139, "max_ms": 52.996, "p95_ms": 46.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.943, "count": 139, "max_ms": 78.673, "p95_ms": 75.437}, "yolo11_640_fire_11class": {"avg_ms": 46.324, "count": 139, "max_ms": 65.105, "p95_ms": 57.794}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.364, "count": 139, "max_ms": 80.586, "p95_ms": 76.162}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.343, "count": 139, "max_ms": 245.64, "p95_ms": 235.978}, "read_interval": {"avg_ms": 216.056, "count": 139, "max_ms": 255.242, "p95_ms": 245.633}, "triton_wait": {"avg_ms": 61.612, "count": 139, "max_ms": 80.598, "p95_ms": 76.361}, "visualize": {"avg_ms": 9.84, "count": 139, "max_ms": 16.242, "p95_ms": 11.269}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.849, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 235}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.949, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:46:09+09:00", "last_error_summary": "[h264 @ 0x55937d706ec0] error while decoding MB 78 32, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:46:09+09:00", "last_error_summary": "[h264 @ 0x55937d706ec0] error while decoding MB 78 32, bytestream -15", "last_exit_at": "2026-07-01T13:46:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.853, "last_started_at": "2026-07-01T13:44:02+09:00", "pid": 10871, "process": {"exists": false, "pid": 10871}, "restart_count": 55, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.989, "sample_count": 1330, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28277, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:46:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:46:09+09:00", "last_frame_read_age_sec": 0.215, "last_frame_read_at": "2026-07-01T13:46:09+09:00", "mjpeg": {"age_sec": 0.206, "exists": true, "mtime": "2026-07-01T13:46:09.298324+09:00", "mtime_epoch": 1782881169.2983243, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:46:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.055, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.634, "pushed": 4.634, "read": 4.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.747, "count": 139, "max_ms": 14.863, "p95_ms": 12.75}, "gstreamer_push": {"avg_ms": 0.878, "count": 139, "max_ms": 1.053, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.914, "count": 139, "max_ms": 52.996, "p95_ms": 46.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.816, "count": 139, "max_ms": 78.673, "p95_ms": 75.317}, "yolo11_640_fire_11class": {"avg_ms": 46.193, "count": 139, "max_ms": 61.609, "p95_ms": 57.775}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.247, "count": 139, "max_ms": 80.586, "p95_ms": 76.162}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.101, "count": 139, "max_ms": 245.64, "p95_ms": 234.938}, "read_interval": {"avg_ms": 216.056, "count": 139, "max_ms": 255.242, "p95_ms": 245.633}, "triton_wait": {"avg_ms": 61.484, "count": 139, "max_ms": 80.598, "p95_ms": 76.173}, "visualize": {"avg_ms": 9.854, "count": 139, "max_ms": 16.242, "p95_ms": 11.324}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 235}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5953.992, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:46:09+09:00", "last_error_summary": "[h264 @ 0x55937d706ec0] error while decoding MB 78 32, bytestream -15", "last_exit_at": "2026-07-01T13:46:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.857, "last_started_at": "2026-07-01T13:44:02+09:00", "pid": 10871, "process": {"exists": false, "pid": 10871}, "restart_count": 55, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.989, "sample_count": 1330, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28277, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:46:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:46:09+09:00", "last_frame_read_age_sec": 0.219, "last_frame_read_at": "2026-07-01T13:46:09+09:00", "mjpeg": {"age_sec": 0.21, "exists": true, "mtime": "2026-07-01T13:46:09.298324+09:00", "mtime_epoch": 1782881169.2983243, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:46:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.055, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.634, "pushed": 4.634, "read": 4.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.747, "count": 139, "max_ms": 14.863, "p95_ms": 12.75}, "gstreamer_push": {"avg_ms": 0.878, "count": 139, "max_ms": 1.053, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.914, "count": 139, "max_ms": 52.996, "p95_ms": 46.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.816, "count": 139, "max_ms": 78.673, "p95_ms": 75.317}, "yolo11_640_fire_11class": {"avg_ms": 46.193, "count": 139, "max_ms": 61.609, "p95_ms": 57.775}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.247, "count": 139, "max_ms": 80.586, "p95_ms": 76.162}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.101, "count": 139, "max_ms": 245.64, "p95_ms": 234.938}, "read_interval": {"avg_ms": 216.056, "count": 139, "max_ms": 255.242, "p95_ms": 245.633}, "triton_wait": {"avg_ms": 61.484, "count": 139, "max_ms": 80.598, "p95_ms": 76.173}, "visualize": {"avg_ms": 9.854, "count": 139, "max_ms": 16.242, "p95_ms": 11.324}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 235}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6064.703, "limit_mb": 8796093022207.996}, "detail": {"pid": 10978, "restart_count": 56, "start_count": 57}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:46:09+09:00", "last_error_summary": "[h264 @ 0x55937d706ec0] error while decoding MB 78 32, bytestream -15", "last_exit_at": "2026-07-01T13:46:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:46:12+09:00", "pid": 10978, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.512, "name": "ffmpeg", "pid": 10978, "rss_mb": 141.512, "state": "S (sleeping)", "threads": 106}, "restart_count": 56, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.429, "sample_count": 1310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28277, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:46:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.579, "last_frame_pushed_at": "2026-07-01T13:46:09+09:00", "last_frame_read_age_sec": 4.791, "last_frame_read_at": "2026-07-01T13:46:09+09:00", "mjpeg": {"age_sec": 4.562, "exists": true, "mtime": "2026-07-01T13:46:09.518328+09:00", "mtime_epoch": 1782881169.5183282, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192253}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:46:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.055, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.99, "pushed": 3.99, "read": 3.957}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.693, "count": 119, "max_ms": 14.71, "p95_ms": 12.75}, "gstreamer_push": {"avg_ms": 0.881, "count": 119, "max_ms": 1.053, "p95_ms": 0.936}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.109, "count": 119, "max_ms": 52.996, "p95_ms": 46.433}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.889, "count": 119, "max_ms": 78.673, "p95_ms": 75.317}, "yolo11_640_fire_11class": {"avg_ms": 46.454, "count": 119, "max_ms": 61.609, "p95_ms": 57.794}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.429, "count": 119, "max_ms": 80.586, "p95_ms": 76.162}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.369, "count": 119, "max_ms": 245.64, "p95_ms": 234.938}, "read_interval": {"avg_ms": 214.012, "count": 118, "max_ms": 255.242, "p95_ms": 245.576}, "triton_wait": {"avg_ms": 61.672, "count": 119, "max_ms": 80.598, "p95_ms": 76.173}, "visualize": {"avg_ms": 9.727, "count": 119, "max_ms": 12.788, "p95_ms": 11.269}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.823, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5955.914, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d463357380] error while decoding MB 34 5, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:48:21+09:00", "last_error_summary": "[h264 @ 0x55d463357380] error while decoding MB 34 5, bytestream -5", "last_exit_at": "2026-07-01T13:46:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.5, "last_started_at": "2026-07-01T13:46:12+09:00", "pid": 10978, "process": {"exists": false, "pid": 10978}, "restart_count": 56, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.078, "sample_count": 1348, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28869, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:48:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-01T13:48:21+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-01T13:48:21+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-01T13:48:21.528764+09:00", "mtime_epoch": 1782881301.5287645, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:48:21+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.312, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.636, "pushed": 4.636, "read": 4.636}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.417, "count": 138, "max_ms": 12.024, "p95_ms": 10.237}, "gstreamer_push": {"avg_ms": 0.866, "count": 139, "max_ms": 1.405, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.919, "count": 138, "max_ms": 49.722, "p95_ms": 42.901}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.018, "count": 138, "max_ms": 78.441, "p95_ms": 72.384}, "yolo11_640_fire_11class": {"avg_ms": 47.156, "count": 138, "max_ms": 65.01, "p95_ms": 60.241}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.544, "count": 138, "max_ms": 79.109, "p95_ms": 74.82}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.792, "count": 139, "max_ms": 242.686, "p95_ms": 226.943}, "read_interval": {"avg_ms": 216.714, "count": 139, "max_ms": 252.184, "p95_ms": 236.783}, "triton_wait": {"avg_ms": 60.768, "count": 138, "max_ms": 79.121, "p95_ms": 76.306}, "visualize": {"avg_ms": 9.813, "count": 138, "max_ms": 11.246, "p95_ms": 10.888}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5955.117, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:48:21+09:00", "last_error_summary": "[h264 @ 0x55d463357380] error while decoding MB 34 5, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:48:21+09:00", "last_error_summary": "[h264 @ 0x55d463357380] error while decoding MB 34 5, bytestream -5", "last_exit_at": "2026-07-01T13:48:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.678, "last_started_at": "2026-07-01T13:46:12+09:00", "pid": 10978, "process": {"exists": false, "pid": 10978}, "restart_count": 56, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.079, "sample_count": 1348, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28870, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:48:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T13:48:21+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-01T13:48:21+09:00", "mjpeg": {"age_sec": 0.229, "exists": true, "mtime": "2026-07-01T13:48:21.528764+09:00", "mtime_epoch": 1782881301.5287645, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:48:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.312, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.641, "pushed": 4.641, "read": 4.608}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.448, "count": 138, "max_ms": 13.398, "p95_ms": 10.263}, "gstreamer_push": {"avg_ms": 0.866, "count": 139, "max_ms": 1.405, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.902, "count": 138, "max_ms": 49.722, "p95_ms": 42.901}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.037, "count": 138, "max_ms": 78.441, "p95_ms": 72.384}, "yolo11_640_fire_11class": {"avg_ms": 47.164, "count": 138, "max_ms": 65.01, "p95_ms": 60.241}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.549, "count": 138, "max_ms": 79.109, "p95_ms": 74.82}}, "truncated_count": 0}, "processing_total": {"avg_ms": 207.001, "count": 139, "max_ms": 242.686, "p95_ms": 228.02}, "read_interval": {"avg_ms": 216.757, "count": 138, "max_ms": 252.184, "p95_ms": 236.783}, "triton_wait": {"avg_ms": 60.773, "count": 138, "max_ms": 79.121, "p95_ms": 76.306}, "visualize": {"avg_ms": 9.836, "count": 138, "max_ms": 12.75, "p95_ms": 10.901}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5955.109, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:48:21+09:00", "last_error_summary": "[h264 @ 0x55d463357380] error while decoding MB 34 5, bytestream -5", "last_exit_at": "2026-07-01T13:48:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.682, "last_started_at": "2026-07-01T13:46:12+09:00", "pid": 10978, "process": {"exists": false, "pid": 10978}, "restart_count": 56, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.079, "sample_count": 1348, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28870, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:48:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T13:48:21+09:00", "last_frame_read_age_sec": 0.24, "last_frame_read_at": "2026-07-01T13:48:21+09:00", "mjpeg": {"age_sec": 0.232, "exists": true, "mtime": "2026-07-01T13:48:21.528764+09:00", "mtime_epoch": 1782881301.5287645, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:48:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.312, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.641, "pushed": 4.641, "read": 4.607}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.448, "count": 138, "max_ms": 13.398, "p95_ms": 10.263}, "gstreamer_push": {"avg_ms": 0.866, "count": 139, "max_ms": 1.405, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.902, "count": 138, "max_ms": 49.722, "p95_ms": 42.901}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.037, "count": 138, "max_ms": 78.441, "p95_ms": 72.384}, "yolo11_640_fire_11class": {"avg_ms": 47.164, "count": 138, "max_ms": 65.01, "p95_ms": 60.241}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.549, "count": 138, "max_ms": 79.109, "p95_ms": 74.82}}, "truncated_count": 0}, "processing_total": {"avg_ms": 207.001, "count": 139, "max_ms": 242.686, "p95_ms": 228.02}, "read_interval": {"avg_ms": 216.757, "count": 138, "max_ms": 252.184, "p95_ms": 236.783}, "triton_wait": {"avg_ms": 60.773, "count": 138, "max_ms": 79.121, "p95_ms": 76.306}, "visualize": {"avg_ms": 9.836, "count": 138, "max_ms": 12.75, "p95_ms": 10.901}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6060.543, "limit_mb": 8796093022207.996}, "detail": {"pid": 11085, "restart_count": 57, "start_count": 58}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:48:21+09:00", "last_error_summary": "[h264 @ 0x55d463357380] error while decoding MB 34 5, bytestream -5", "last_exit_at": "2026-07-01T13:48:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:48:24+09:00", "pid": 11085, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.785, "name": "ffmpeg", "pid": 11085, "rss_mb": 136.785, "state": "S (sleeping)", "threads": 106}, "restart_count": 57, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.202, "sample_count": 1328, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28870, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:48:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.329, "last_frame_pushed_at": "2026-07-01T13:48:21+09:00", "last_frame_read_age_sec": 4.561, "last_frame_read_at": "2026-07-01T13:48:21+09:00", "mjpeg": {"age_sec": 4.314, "exists": true, "mtime": "2026-07-01T13:48:21.768769+09:00", "mtime_epoch": 1782881301.768769, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199126}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:48:26+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.312, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.934, "pushed": 3.934, "read": 3.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.492, "count": 118, "max_ms": 13.398, "p95_ms": 10.635}, "gstreamer_push": {"avg_ms": 0.87, "count": 118, "max_ms": 1.405, "p95_ms": 0.913}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.759, "count": 118, "max_ms": 47.892, "p95_ms": 42.367}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.742, "count": 118, "max_ms": 78.441, "p95_ms": 72.384}, "yolo11_640_fire_11class": {"avg_ms": 47.04, "count": 118, "max_ms": 65.01, "p95_ms": 60.241}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.227, "count": 118, "max_ms": 79.109, "p95_ms": 74.82}}, "truncated_count": 0}, "processing_total": {"avg_ms": 207.639, "count": 118, "max_ms": 242.686, "p95_ms": 230.617}, "read_interval": {"avg_ms": 217.48, "count": 118, "max_ms": 252.184, "p95_ms": 238.247}, "triton_wait": {"avg_ms": 60.452, "count": 118, "max_ms": 79.121, "p95_ms": 76.306}, "visualize": {"avg_ms": 9.881, "count": 118, "max_ms": 12.75, "p95_ms": 10.915}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 245}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5957.18, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5631da0820c0] error while decoding MB 44 15, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:50:30+09:00", "last_error_summary": "[h264 @ 0x5631da0820c0] error while decoding MB 44 15, bytestream -5", "last_exit_at": "2026-07-01T13:48:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.479, "last_started_at": "2026-07-01T13:48:24+09:00", "pid": 11085, "process": {"exists": false, "pid": 11085}, "restart_count": 57, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.534, "sample_count": 1360, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:50:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T13:50:30+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T13:50:30+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T13:50:30.519177+09:00", "mtime_epoch": 1782881430.5191767, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206526}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:50:30+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.57, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.701, "pushed": 4.701, "read": 4.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.078, "count": 140, "max_ms": 12.688, "p95_ms": 10.009}, "gstreamer_push": {"avg_ms": 0.872, "count": 141, "max_ms": 1.418, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.689, "count": 140, "max_ms": 47.587, "p95_ms": 44.446}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.187, "count": 140, "max_ms": 78.284, "p95_ms": 72.958}, "yolo11_640_fire_11class": {"avg_ms": 47.921, "count": 140, "max_ms": 66.481, "p95_ms": 60.474}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.079, "count": 140, "max_ms": 81.68, "p95_ms": 73.459}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.604, "count": 141, "max_ms": 224.986, "p95_ms": 218.964}, "read_interval": {"avg_ms": 213.824, "count": 141, "max_ms": 236.137, "p95_ms": 227.001}, "triton_wait": {"avg_ms": 61.431, "count": 140, "max_ms": 81.691, "p95_ms": 75.371}, "visualize": {"avg_ms": 9.711, "count": 140, "max_ms": 13.509, "p95_ms": 10.134}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 245}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5956.711, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:50:30+09:00", "last_error_summary": "[h264 @ 0x5631da0820c0] error while decoding MB 44 15, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:50:30+09:00", "last_error_summary": "[h264 @ 0x5631da0820c0] error while decoding MB 44 15, bytestream -5", "last_exit_at": "2026-07-01T13:50:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.64, "last_started_at": "2026-07-01T13:48:24+09:00", "pid": 11085, "process": {"exists": false, "pid": 11085}, "restart_count": 57, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.405, "sample_count": 1360, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29457, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:50:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:50:30+09:00", "last_frame_read_age_sec": 0.215, "last_frame_read_at": "2026-07-01T13:50:30+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-01T13:50:30.519177+09:00", "mtime_epoch": 1782881430.5191767, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206526}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:50:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.57, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.708, "pushed": 4.708, "read": 4.675}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.088, "count": 140, "max_ms": 12.688, "p95_ms": 10.077}, "gstreamer_push": {"avg_ms": 0.872, "count": 141, "max_ms": 1.418, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.666, "count": 140, "max_ms": 47.587, "p95_ms": 44.446}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.16, "count": 140, "max_ms": 78.284, "p95_ms": 72.958}, "yolo11_640_fire_11class": {"avg_ms": 47.88, "count": 140, "max_ms": 66.481, "p95_ms": 60.474}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.075, "count": 140, "max_ms": 81.68, "p95_ms": 73.459}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.595, "count": 141, "max_ms": 224.986, "p95_ms": 218.964}, "read_interval": {"avg_ms": 213.772, "count": 140, "max_ms": 236.137, "p95_ms": 227.001}, "triton_wait": {"avg_ms": 61.427, "count": 140, "max_ms": 81.691, "p95_ms": 75.371}, "visualize": {"avg_ms": 9.732, "count": 140, "max_ms": 13.509, "p95_ms": 10.179}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 245}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5956.703, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:50:30+09:00", "last_error_summary": "[h264 @ 0x5631da0820c0] error while decoding MB 44 15, bytestream -5", "last_exit_at": "2026-07-01T13:50:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.644, "last_started_at": "2026-07-01T13:48:24+09:00", "pid": 11085, "process": {"exists": false, "pid": 11085}, "restart_count": 57, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.405, "sample_count": 1360, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29457, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:50:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:50:30+09:00", "last_frame_read_age_sec": 0.219, "last_frame_read_at": "2026-07-01T13:50:30+09:00", "mjpeg": {"age_sec": 0.207, "exists": true, "mtime": "2026-07-01T13:50:30.519177+09:00", "mtime_epoch": 1782881430.5191767, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206526}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:50:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.57, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.708, "pushed": 4.708, "read": 4.674}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.088, "count": 140, "max_ms": 12.688, "p95_ms": 10.077}, "gstreamer_push": {"avg_ms": 0.872, "count": 141, "max_ms": 1.418, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.666, "count": 140, "max_ms": 47.587, "p95_ms": 44.446}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.16, "count": 140, "max_ms": 78.284, "p95_ms": 72.958}, "yolo11_640_fire_11class": {"avg_ms": 47.88, "count": 140, "max_ms": 66.481, "p95_ms": 60.474}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.075, "count": 140, "max_ms": 81.68, "p95_ms": 73.459}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.595, "count": 141, "max_ms": 224.986, "p95_ms": 218.964}, "read_interval": {"avg_ms": 213.772, "count": 140, "max_ms": 236.137, "p95_ms": 227.001}, "triton_wait": {"avg_ms": 61.427, "count": 140, "max_ms": 81.691, "p95_ms": 75.371}, "visualize": {"avg_ms": 9.732, "count": 140, "max_ms": 13.509, "p95_ms": 10.179}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 245}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6067.938, "limit_mb": 8796093022207.996}, "detail": {"pid": 11192, "restart_count": 58, "start_count": 59}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:50:30+09:00", "last_error_summary": "[h264 @ 0x5631da0820c0] error while decoding MB 44 15, bytestream -5", "last_exit_at": "2026-07-01T13:50:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:50:33+09:00", "pid": 11192, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.977, "name": "ffmpeg", "pid": 11192, "rss_mb": 141.977, "state": "S (sleeping)", "threads": 106}, "restart_count": 58, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.342, "sample_count": 1338, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29457, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:50:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.813, "last_frame_pushed_at": "2026-07-01T13:50:30+09:00", "last_frame_read_age_sec": 5.025, "last_frame_read_at": "2026-07-01T13:50:30+09:00", "mjpeg": {"age_sec": 4.793, "exists": true, "mtime": "2026-07-01T13:50:30.739181+09:00", "mtime_epoch": 1782881430.739181, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:50:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.57, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.945, "pushed": 3.945, "read": 3.912}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.114, "count": 117, "max_ms": 12.688, "p95_ms": 10.317}, "gstreamer_push": {"avg_ms": 0.878, "count": 118, "max_ms": 1.418, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.749, "count": 117, "max_ms": 47.587, "p95_ms": 44.819}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.514, "count": 117, "max_ms": 78.284, "p95_ms": 73.066}, "yolo11_640_fire_11class": {"avg_ms": 48.265, "count": 117, "max_ms": 66.481, "p95_ms": 60.691}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.365, "count": 117, "max_ms": 81.68, "p95_ms": 75.358}}, "truncated_count": 0}, "processing_total": {"avg_ms": 205.228, "count": 118, "max_ms": 224.986, "p95_ms": 219.409}, "read_interval": {"avg_ms": 214.423, "count": 117, "max_ms": 236.137, "p95_ms": 228.355}, "triton_wait": {"avg_ms": 61.756, "count": 117, "max_ms": 81.691, "p95_ms": 75.581}, "visualize": {"avg_ms": 9.758, "count": 117, "max_ms": 13.509, "p95_ms": 10.19}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5957.582, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5580ea25a140] error while decoding MB 36 33, bytestream -38", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:52:39+09:00", "last_error_summary": "[h264 @ 0x5580ea25a140] error while decoding MB 36 33, bytestream -38", "last_exit_at": "2026-07-01T13:50:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.348, "last_started_at": "2026-07-01T13:50:33+09:00", "pid": 11192, "process": {"exists": false, "pid": 11192}, "restart_count": 58, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.875, "sample_count": 1369, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30045, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:52:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T13:52:39+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T13:52:39+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T13:52:39.833576+09:00", "mtime_epoch": 1782881559.8335762, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206789}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:52:39+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.828, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.696, "pushed": 4.696, "read": 4.696}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.196, "count": 140, "max_ms": 13.933, "p95_ms": 10.731}, "gstreamer_push": {"avg_ms": 0.87, "count": 140, "max_ms": 1.246, "p95_ms": 0.91}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.285, "count": 140, "max_ms": 54.074, "p95_ms": 41.772}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.493, "count": 140, "max_ms": 80.599, "p95_ms": 68.042}, "yolo11_640_fire_11class": {"avg_ms": 45.931, "count": 140, "max_ms": 60.618, "p95_ms": 56.216}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.659, "count": 140, "max_ms": 83.546, "p95_ms": 78.983}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.153, "count": 140, "max_ms": 240.094, "p95_ms": 223.975}, "read_interval": {"avg_ms": 214.045, "count": 140, "max_ms": 250.633, "p95_ms": 234.353}, "triton_wait": {"avg_ms": 59.905, "count": 140, "max_ms": 83.558, "p95_ms": 78.995}, "visualize": {"avg_ms": 9.775, "count": 140, "max_ms": 14.346, "p95_ms": 11.044}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.816, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5957.211, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:52:39+09:00", "last_error_summary": "[h264 @ 0x5580ea25a140] error while decoding MB 36 33, bytestream -38", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:52:39+09:00", "last_error_summary": "[h264 @ 0x5580ea25a140] error while decoding MB 36 33, bytestream -38", "last_exit_at": "2026-07-01T13:52:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.505, "last_started_at": "2026-07-01T13:50:33+09:00", "pid": 11192, "process": {"exists": false, "pid": 11192}, "restart_count": 58, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.875, "sample_count": 1369, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30046, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:52:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:52:40+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T13:52:39+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-01T13:52:39.833576+09:00", "mtime_epoch": 1782881559.8335762, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206789}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:52:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.828, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.704, "pushed": 4.704, "read": 4.671}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.197, "count": 140, "max_ms": 13.933, "p95_ms": 10.731}, "gstreamer_push": {"avg_ms": 0.869, "count": 141, "max_ms": 1.246, "p95_ms": 0.91}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.279, "count": 140, "max_ms": 54.074, "p95_ms": 41.772}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.507, "count": 140, "max_ms": 80.599, "p95_ms": 68.042}, "yolo11_640_fire_11class": {"avg_ms": 45.981, "count": 140, "max_ms": 60.618, "p95_ms": 56.216}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.648, "count": 140, "max_ms": 83.546, "p95_ms": 78.983}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.16, "count": 141, "max_ms": 240.094, "p95_ms": 223.975}, "read_interval": {"avg_ms": 214.045, "count": 140, "max_ms": 250.633, "p95_ms": 234.353}, "triton_wait": {"avg_ms": 59.898, "count": 140, "max_ms": 83.558, "p95_ms": 78.995}, "visualize": {"avg_ms": 9.776, "count": 140, "max_ms": 14.346, "p95_ms": 11.044}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5957.203, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:52:39+09:00", "last_error_summary": "[h264 @ 0x5580ea25a140] error while decoding MB 36 33, bytestream -38", "last_exit_at": "2026-07-01T13:52:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.508, "last_started_at": "2026-07-01T13:50:33+09:00", "pid": 11192, "process": {"exists": false, "pid": 11192}, "restart_count": 58, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.875, "sample_count": 1369, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30046, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:52:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:52:40+09:00", "last_frame_read_age_sec": 0.213, "last_frame_read_at": "2026-07-01T13:52:39+09:00", "mjpeg": {"age_sec": 0.206, "exists": true, "mtime": "2026-07-01T13:52:39.833576+09:00", "mtime_epoch": 1782881559.8335762, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206789}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:52:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.828, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.704, "pushed": 4.704, "read": 4.671}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.197, "count": 140, "max_ms": 13.933, "p95_ms": 10.731}, "gstreamer_push": {"avg_ms": 0.869, "count": 141, "max_ms": 1.246, "p95_ms": 0.91}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.279, "count": 140, "max_ms": 54.074, "p95_ms": 41.772}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.507, "count": 140, "max_ms": 80.599, "p95_ms": 68.042}, "yolo11_640_fire_11class": {"avg_ms": 45.981, "count": 140, "max_ms": 60.618, "p95_ms": 56.216}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.648, "count": 140, "max_ms": 83.546, "p95_ms": 78.983}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.16, "count": 141, "max_ms": 240.094, "p95_ms": 223.975}, "read_interval": {"avg_ms": 214.045, "count": 140, "max_ms": 250.633, "p95_ms": 234.353}, "triton_wait": {"avg_ms": 59.898, "count": 140, "max_ms": 83.558, "p95_ms": 78.995}, "visualize": {"avg_ms": 9.776, "count": 140, "max_ms": 14.346, "p95_ms": 11.044}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6065.438, "limit_mb": 8796093022207.996}, "detail": {"pid": 11299, "restart_count": 59, "start_count": 60}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:52:39+09:00", "last_error_summary": "[h264 @ 0x5580ea25a140] error while decoding MB 36 33, bytestream -38", "last_exit_at": "2026-07-01T13:52:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:52:42+09:00", "pid": 11299, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.953, "name": "ffmpeg", "pid": 11299, "rss_mb": 138.953, "state": "S (sleeping)", "threads": 106}, "restart_count": 59, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.591, "sample_count": 1348, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30046, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:52:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.567, "last_frame_pushed_at": "2026-07-01T13:52:40+09:00", "last_frame_read_age_sec": 4.773, "last_frame_read_at": "2026-07-01T13:52:39+09:00", "mjpeg": {"age_sec": 4.55, "exists": true, "mtime": "2026-07-01T13:52:40.049580+09:00", "mtime_epoch": 1782881560.04958, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207048}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:52:44+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2364.828, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.016, "pushed": 4.016, "read": 3.982}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.117, "count": 119, "max_ms": 13.933, "p95_ms": 10.852}, "gstreamer_push": {"avg_ms": 0.87, "count": 120, "max_ms": 1.246, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.558, "count": 119, "max_ms": 54.074, "p95_ms": 42.08}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.435, "count": 119, "max_ms": 80.599, "p95_ms": 73.133}, "yolo11_640_fire_11class": {"avg_ms": 45.961, "count": 119, "max_ms": 60.618, "p95_ms": 56.748}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.523, "count": 119, "max_ms": 83.546, "p95_ms": 79.824}}, "truncated_count": 0}, "processing_total": {"avg_ms": 202.76, "count": 120, "max_ms": 225.145, "p95_ms": 223.075}, "read_interval": {"avg_ms": 212.72, "count": 119, "max_ms": 235.802, "p95_ms": 233.819}, "triton_wait": {"avg_ms": 59.809, "count": 119, "max_ms": 83.558, "p95_ms": 80.236}, "visualize": {"avg_ms": 9.734, "count": 119, "max_ms": 14.346, "p95_ms": 12.403}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_115104_ab775f64.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 12, "tracking_summary": 605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6199.176, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:45:59+09:00", "last_error_summary": "[h264 @ 0x5589545bbb40] error while decoding MB 73 65, bytestream -29", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 4075.252, "last_started_at": "2026-07-01T12:46:02+09:00", "pid": 7650, "process": {"exists": true, "fd_count": 5, "hwm_mb": 331.414, "name": "ffmpeg", "pid": 7650, "rss_mb": 331.414, "state": "S (sleeping)", "threads": 106}, "restart_count": 1, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.047, "sample_count": 2561, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72715, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:53:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "ab775f64", "last_frame_pushed_age_sec": 30.059, "last_frame_pushed_at": "2026-07-01T13:53:27+09:00", "last_frame_read_age_sec": 30.12, "last_frame_read_at": "2026-07-01T13:53:27+09:00", "mjpeg": {"age_sec": 30.043, "exists": true, "mtime": "2026-07-01T13:53:27.766461+09:00", "mtime_epoch": 1782881607.7664614, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T13:53:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1648.73, "name": "celery", "pid": 267, "rss_mb": 1313.16, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:51:04+09:00", "state": "running", "task_id": "64e6ede9-0997-463c-b342-3af5d237a507", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_115104_ab775f64.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 12, "tracking_summary": 605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6189.223, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T12:45:59+09:00", "last_error_summary": "[h264 @ 0x5589545bbb40] error while decoding MB 73 65, bytestream -29", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 4075.266, "last_started_at": "2026-07-01T12:46:02+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 1, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.047, "sample_count": 2561, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72715, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:53:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "ab775f64", "last_frame_pushed_age_sec": 30.073, "last_frame_pushed_at": "2026-07-01T13:53:27+09:00", "last_frame_read_age_sec": 30.134, "last_frame_read_at": "2026-07-01T13:53:27+09:00", "mjpeg": {"age_sec": 30.057, "exists": true, "mtime": "2026-07-01T13:53:27.766461+09:00", "mtime_epoch": 1782881607.7664614, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T13:53:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1648.73, "name": "celery", "pid": 267, "rss_mb": 1313.16, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:51:04+09:00", "state": "running", "task_id": "64e6ede9-0997-463c-b342-3af5d237a507", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_115104_ab775f64.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 12, "tracking_summary": 605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5896.051, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55dd347a9e80] error while decoding MB 36 56, bytestream -11 [h264 @ 0x55dd34878d00] error while decoding MB 91 31, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:53:57+09:00", "last_error_summary": "[h264 @ 0x55dd347a9e80] error while decoding MB 36 56, bytestream -11 [h264 @ 0x55dd34878d00] error while decoding MB 91 31, bytestream -15", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 4075.311, "last_started_at": "2026-07-01T12:46:02+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 1, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.047, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72715, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:53:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "ab775f64", "last_frame_pushed_age_sec": 30.118, "last_frame_pushed_at": "2026-07-01T13:53:27+09:00", "last_frame_read_age_sec": 30.178, "last_frame_read_at": "2026-07-01T13:53:27+09:00", "mjpeg": {"age_sec": 30.102, "exists": true, "mtime": "2026-07-01T13:53:27.766461+09:00", "mtime_epoch": 1782881607.7664614, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T13:53:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1648.73, "name": "celery", "pid": 267, "rss_mb": 1313.16, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:51:04+09:00", "state": "running", "task_id": "64e6ede9-0997-463c-b342-3af5d237a507", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_115104_ab775f64.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 12, "tracking_summary": 605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 6019.047, "limit_mb": 8796093022207.996}, "detail": {"pid": 11406, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:53:57+09:00", "last_error_summary": "[h264 @ 0x55dd347a9e80] error while decoding MB 36 56, bytestream -11 [h264 @ 0x55dd34878d00] error while decoding MB 91 31, bytestream -15", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:54:00+09:00", "pid": 11406, "process": {"exists": true, "fd_count": 5, "hwm_mb": 154.723, "name": "ffmpeg", "pid": 11406, "rss_mb": 154.723, "state": "S (sleeping)", "threads": 49}, "restart_count": 2, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.324, "sample_count": 2510, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72715, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:53:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "ab775f64", "last_frame_pushed_age_sec": 34.703, "last_frame_pushed_at": "2026-07-01T13:53:27+09:00", "last_frame_read_age_sec": 34.763, "last_frame_read_at": "2026-07-01T13:53:27+09:00", "mjpeg": {"age_sec": 34.687, "exists": true, "mtime": "2026-07-01T13:53:27.766461+09:00", "mtime_epoch": 1782881607.7664614, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T13:54:02+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1648.73, "name": "celery", "pid": 267, "rss_mb": 1313.16, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T11:51:04+09:00", "state": "running", "task_id": "64e6ede9-0997-463c-b342-3af5d237a507", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 255}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5962.621, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55bd7f6e7180] error while decoding MB 73 31, bytestream -21", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:54:50+09:00", "last_error_summary": "[h264 @ 0x55bd7f6e7180] error while decoding MB 73 31, bytestream -21", "last_exit_at": "2026-07-01T13:52:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.377, "last_started_at": "2026-07-01T13:52:42+09:00", "pid": 11299, "process": {"exists": false, "pid": 11299}, "restart_count": 59, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.078, "sample_count": 1389, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:54:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T13:54:50+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T13:54:50+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T13:54:50.931993+09:00", "mtime_epoch": 1782881690.9319932, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200117}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:54:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.086, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.901, "pushed": 4.901, "read": 4.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.907, "count": 147, "max_ms": 11.879, "p95_ms": 10.708}, "gstreamer_push": {"avg_ms": 0.864, "count": 147, "max_ms": 1.144, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.029, "count": 147, "max_ms": 59.305, "p95_ms": 46.913}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.566, "count": 147, "max_ms": 86.836, "p95_ms": 70.19}, "yolo11_640_fire_11class": {"avg_ms": 44.493, "count": 147, "max_ms": 59.834, "p95_ms": 52.99}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.321, "count": 147, "max_ms": 89.336, "p95_ms": 80.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.81, "count": 147, "max_ms": 269.664, "p95_ms": 222.114}, "read_interval": {"avg_ms": 203.633, "count": 148, "max_ms": 279.164, "p95_ms": 232.155}, "triton_wait": {"avg_ms": 59.367, "count": 147, "max_ms": 89.347, "p95_ms": 80.536}, "visualize": {"avg_ms": 9.238, "count": 147, "max_ms": 12.297, "p95_ms": 10.64}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 255}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5962.117, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:54:50+09:00", "last_error_summary": "[h264 @ 0x55bd7f6e7180] error while decoding MB 73 31, bytestream -21", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:54:50+09:00", "last_error_summary": "[h264 @ 0x55bd7f6e7180] error while decoding MB 73 31, bytestream -21", "last_exit_at": "2026-07-01T13:54:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.537, "last_started_at": "2026-07-01T13:52:42+09:00", "pid": 11299, "process": {"exists": false, "pid": 11299}, "restart_count": 59, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.078, "sample_count": 1389, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:54:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:54:51+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T13:54:50+09:00", "mjpeg": {"age_sec": 0.204, "exists": true, "mtime": "2026-07-01T13:54:50.931993+09:00", "mtime_epoch": 1782881690.9319932, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200117}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:54:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.086, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.938, "pushed": 4.938, "read": 4.905}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.909, "count": 147, "max_ms": 11.879, "p95_ms": 10.708}, "gstreamer_push": {"avg_ms": 0.864, "count": 148, "max_ms": 1.144, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.0, "count": 147, "max_ms": 59.305, "p95_ms": 46.913}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.533, "count": 147, "max_ms": 86.836, "p95_ms": 70.19}, "yolo11_640_fire_11class": {"avg_ms": 44.483, "count": 147, "max_ms": 59.834, "p95_ms": 52.99}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.303, "count": 147, "max_ms": 89.336, "p95_ms": 80.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.889, "count": 148, "max_ms": 269.664, "p95_ms": 222.114}, "read_interval": {"avg_ms": 203.699, "count": 147, "max_ms": 279.164, "p95_ms": 232.155}, "triton_wait": {"avg_ms": 59.349, "count": 147, "max_ms": 89.347, "p95_ms": 80.536}, "visualize": {"avg_ms": 9.245, "count": 147, "max_ms": 12.297, "p95_ms": 10.64}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 255}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5962.109, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:54:50+09:00", "last_error_summary": "[h264 @ 0x55bd7f6e7180] error while decoding MB 73 31, bytestream -21", "last_exit_at": "2026-07-01T13:54:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.541, "last_started_at": "2026-07-01T13:52:42+09:00", "pid": 11299, "process": {"exists": false, "pid": 11299}, "restart_count": 59, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.078, "sample_count": 1389, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:54:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T13:54:51+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T13:54:50+09:00", "mjpeg": {"age_sec": 0.208, "exists": true, "mtime": "2026-07-01T13:54:50.931993+09:00", "mtime_epoch": 1782881690.9319932, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200117}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:54:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.086, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.938, "pushed": 4.938, "read": 4.904}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.909, "count": 147, "max_ms": 11.879, "p95_ms": 10.708}, "gstreamer_push": {"avg_ms": 0.864, "count": 148, "max_ms": 1.144, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.0, "count": 147, "max_ms": 59.305, "p95_ms": 46.913}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.533, "count": 147, "max_ms": 86.836, "p95_ms": 70.19}, "yolo11_640_fire_11class": {"avg_ms": 44.483, "count": 147, "max_ms": 59.834, "p95_ms": 52.99}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.303, "count": 147, "max_ms": 89.336, "p95_ms": 80.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.889, "count": 148, "max_ms": 269.664, "p95_ms": 222.114}, "read_interval": {"avg_ms": 203.699, "count": 147, "max_ms": 279.164, "p95_ms": 232.155}, "triton_wait": {"avg_ms": 59.349, "count": 147, "max_ms": 89.347, "p95_ms": 80.536}, "visualize": {"avg_ms": 9.245, "count": 147, "max_ms": 12.297, "p95_ms": 10.64}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 255}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6070.418, "limit_mb": 8796093022207.996}, "detail": {"pid": 11558, "restart_count": 60, "start_count": 61}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:54:50+09:00", "last_error_summary": "[h264 @ 0x55bd7f6e7180] error while decoding MB 73 31, bytestream -21", "last_exit_at": "2026-07-01T13:54:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:54:53+09:00", "pid": 11558, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.074, "name": "ffmpeg", "pid": 11558, "rss_mb": 139.074, "state": "S (sleeping)", "threads": 106}, "restart_count": 60, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.079, "sample_count": 1366, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:54:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.766, "last_frame_pushed_at": "2026-07-01T13:54:51+09:00", "last_frame_read_age_sec": 4.973, "last_frame_read_at": "2026-07-01T13:54:50+09:00", "mjpeg": {"age_sec": 4.751, "exists": true, "mtime": "2026-07-01T13:54:51.147997+09:00", "mtime_epoch": 1782881691.1479974, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198173}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:54:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.086, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.123, "pushed": 4.123, "read": 4.089}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.952, "count": 123, "max_ms": 11.879, "p95_ms": 10.817}, "gstreamer_push": {"avg_ms": 0.865, "count": 123, "max_ms": 1.144, "p95_ms": 0.912}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.829, "count": 123, "max_ms": 59.13, "p95_ms": 46.913}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.747, "count": 123, "max_ms": 86.836, "p95_ms": 70.028}, "yolo11_640_fire_11class": {"avg_ms": 43.661, "count": 123, "max_ms": 58.596, "p95_ms": 50.771}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.747, "count": 123, "max_ms": 89.336, "p95_ms": 80.136}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.467, "count": 123, "max_ms": 269.664, "p95_ms": 222.593}, "read_interval": {"avg_ms": 205.268, "count": 122, "max_ms": 279.164, "p95_ms": 233.686}, "triton_wait": {"avg_ms": 58.787, "count": 123, "max_ms": 89.347, "p95_ms": 80.147}, "visualize": {"avg_ms": 9.331, "count": 123, "max_ms": 12.297, "p95_ms": 10.705}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.835, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 260}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5962.953, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55fbed9595c0] left block unavailable for requested intra mode [h264 @ 0x55fbed9595c0] error while decoding MB 0 40, bytestream 1419 [h264 @ 0x55fbed90e940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fbed90e940] error while decoding MB 0 18, bytestream 64269 [h264 @ 0x55fbed9fcc80] left block unavailable for requested intra mode [h264 @ 0x55fbed9fcc80] error while decoding MB 0 11, bytestream 4986 [h264 @ 0x55fbee64a000] top block unavailable for requested intra m", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:57:23+09:00", "last_error_summary": "[h264 @ 0x55fbed9595c0] left block unavailable for requested intra mode [h264 @ 0x55fbed9595c0] error while decoding MB 0 40, bytestream 1419 [h264 @ 0x55fbed90e940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fbed90e940] error while decoding MB 0 18, bytestream 64269 [h264 @ 0x55fbed9fcc80] left block unavailable for requested intra mode [h264 @ 0x55fbed9fcc80] error while decoding MB 0 11, bytestream 4986 [h264 @ 0x55fbee64a000] top block unavailable for requested intra m", "last_exit_at": "2026-07-01T13:54:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.867, "last_started_at": "2026-07-01T13:54:53+09:00", "pid": 11558, "process": {"exists": false, "pid": 11558}, "restart_count": 60, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.805, "sample_count": 1277, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31246, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T13:57:23+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T13:57:23+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T13:57:23.722796+09:00", "mtime_epoch": 1782881843.7227957, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200605}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:57:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.344, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 56, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.876, "pushed": 1.876, "read": 1.876}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.082, "count": 56, "max_ms": 11.403, "p95_ms": 10.128}, "gstreamer_push": {"avg_ms": 0.873, "count": 56, "max_ms": 0.935, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.193, "count": 56, "max_ms": 45.966, "p95_ms": 37.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.269, "count": 56, "max_ms": 71.33, "p95_ms": 70.567}, "yolo11_640_fire_11class": {"avg_ms": 46.693, "count": 56, "max_ms": 63.076, "p95_ms": 53.017}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.353, "count": 56, "max_ms": 80.572, "p95_ms": 71.393}}, "truncated_count": 0}, "processing_total": {"avg_ms": 205.834, "count": 56, "max_ms": 221.764, "p95_ms": 214.264}, "read_interval": {"avg_ms": 535.865, "count": 56, "max_ms": 18146.321, "p95_ms": 224.369}, "triton_wait": {"avg_ms": 60.379, "count": 56, "max_ms": 80.583, "p95_ms": 71.405}, "visualize": {"avg_ms": 9.639, "count": 56, "max_ms": 10.217, "p95_ms": 10.108}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.857, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 260}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5962.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:57:23+09:00", "last_error_summary": "[h264 @ 0x55fbed9595c0] left block unavailable for requested intra mode [h264 @ 0x55fbed9595c0] error while decoding MB 0 40, bytestream 1419 [h264 @ 0x55fbed90e940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fbed90e940] error while decoding MB 0 18, bytestream 64269 [h264 @ 0x55fbed9fcc80] left block unavailable for requested intra mode [h264 @ 0x55fbed9fcc80] error while decoding MB 0 11, bytestream 4986 [h264 @ 0x55fbee64a000] top block unavailable for requested intra m", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:57:23+09:00", "last_error_summary": "[h264 @ 0x55fbed9595c0] left block unavailable for requested intra mode [h264 @ 0x55fbed9595c0] error while decoding MB 0 40, bytestream 1419 [h264 @ 0x55fbed90e940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fbed90e940] error while decoding MB 0 18, bytestream 64269 [h264 @ 0x55fbed9fcc80] left block unavailable for requested intra mode [h264 @ 0x55fbed9fcc80] error while decoding MB 0 11, bytestream 4986 [h264 @ 0x55fbee64a000] top block unavailable for requested intra m", "last_exit_at": "2026-07-01T13:57:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.051, "last_started_at": "2026-07-01T13:54:53+09:00", "pid": 11558, "process": {"exists": false, "pid": 11558}, "restart_count": 60, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.805, "sample_count": 1277, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T13:57:23+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T13:57:23+09:00", "mjpeg": {"age_sec": 0.227, "exists": true, "mtime": "2026-07-01T13:57:23.722796+09:00", "mtime_epoch": 1782881843.7227957, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200605}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:57:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.344, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.878, "pushed": 1.878, "read": 1.844}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.129, "count": 56, "max_ms": 12.132, "p95_ms": 10.996}, "gstreamer_push": {"avg_ms": 0.873, "count": 56, "max_ms": 0.935, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.278, "count": 56, "max_ms": 45.966, "p95_ms": 37.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.231, "count": 56, "max_ms": 71.33, "p95_ms": 70.567}, "yolo11_640_fire_11class": {"avg_ms": 46.7, "count": 56, "max_ms": 63.076, "p95_ms": 53.017}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.284, "count": 56, "max_ms": 80.572, "p95_ms": 71.393}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.325, "count": 56, "max_ms": 234.141, "p95_ms": 214.951}, "read_interval": {"avg_ms": 541.677, "count": 55, "max_ms": 18146.321, "p95_ms": 224.369}, "triton_wait": {"avg_ms": 60.31, "count": 56, "max_ms": 80.583, "p95_ms": 71.405}, "visualize": {"avg_ms": 9.789, "count": 56, "max_ms": 17.747, "p95_ms": 10.183}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.824, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 260}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5962.484, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:57:23+09:00", "last_error_summary": "[h264 @ 0x55fbed9595c0] left block unavailable for requested intra mode [h264 @ 0x55fbed9595c0] error while decoding MB 0 40, bytestream 1419 [h264 @ 0x55fbed90e940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fbed90e940] error while decoding MB 0 18, bytestream 64269 [h264 @ 0x55fbed9fcc80] left block unavailable for requested intra mode [h264 @ 0x55fbed9fcc80] error while decoding MB 0 11, bytestream 4986 [h264 @ 0x55fbee64a000] top block unavailable for requested intra m", "last_exit_at": "2026-07-01T13:57:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.055, "last_started_at": "2026-07-01T13:54:53+09:00", "pid": 11558, "process": {"exists": false, "pid": 11558}, "restart_count": 60, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.805, "sample_count": 1277, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T13:57:23+09:00", "last_frame_read_age_sec": 0.241, "last_frame_read_at": "2026-07-01T13:57:23+09:00", "mjpeg": {"age_sec": 0.23, "exists": true, "mtime": "2026-07-01T13:57:23.722796+09:00", "mtime_epoch": 1782881843.7227957, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200605}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:57:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.344, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.877, "pushed": 1.877, "read": 1.844}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.129, "count": 56, "max_ms": 12.132, "p95_ms": 10.996}, "gstreamer_push": {"avg_ms": 0.873, "count": 56, "max_ms": 0.935, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.278, "count": 56, "max_ms": 45.966, "p95_ms": 37.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.231, "count": 56, "max_ms": 71.33, "p95_ms": 70.567}, "yolo11_640_fire_11class": {"avg_ms": 46.7, "count": 56, "max_ms": 63.076, "p95_ms": 53.017}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.284, "count": 56, "max_ms": 80.572, "p95_ms": 71.393}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.325, "count": 56, "max_ms": 234.141, "p95_ms": 214.951}, "read_interval": {"avg_ms": 541.677, "count": 55, "max_ms": 18146.321, "p95_ms": 224.369}, "triton_wait": {"avg_ms": 60.31, "count": 56, "max_ms": 80.583, "p95_ms": 71.405}, "visualize": {"avg_ms": 9.789, "count": 56, "max_ms": 17.747, "p95_ms": 10.183}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.828, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 260}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6071.641, "limit_mb": 8796093022207.996}, "detail": {"pid": 11665, "restart_count": 61, "start_count": 62}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:57:23+09:00", "last_error_summary": "[h264 @ 0x55fbed9595c0] left block unavailable for requested intra mode [h264 @ 0x55fbed9595c0] error while decoding MB 0 40, bytestream 1419 [h264 @ 0x55fbed90e940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fbed90e940] error while decoding MB 0 18, bytestream 64269 [h264 @ 0x55fbed9fcc80] left block unavailable for requested intra mode [h264 @ 0x55fbed9fcc80] error while decoding MB 0 11, bytestream 4986 [h264 @ 0x55fbee64a000] top block unavailable for requested intra m", "last_exit_at": "2026-07-01T13:57:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:57:26+09:00", "pid": 11665, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.336, "name": "ffmpeg", "pid": 11665, "rss_mb": 138.336, "state": "S (sleeping)", "threads": 106}, "restart_count": 61, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.83, "sample_count": 1255, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.758, "last_frame_pushed_at": "2026-07-01T13:57:23+09:00", "last_frame_read_age_sec": 4.993, "last_frame_read_at": "2026-07-01T13:57:23+09:00", "mjpeg": {"age_sec": 4.742, "exists": true, "mtime": "2026-07-01T13:57:23.962800+09:00", "mtime_epoch": 1782881843.9628, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191550}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:57:28+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.344, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 33, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.14, "pushed": 1.14, "read": 1.107}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.024, "count": 34, "max_ms": 12.132, "p95_ms": 9.747}, "gstreamer_push": {"avg_ms": 0.87, "count": 34, "max_ms": 0.935, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.608, "count": 34, "max_ms": 45.966, "p95_ms": 38.972}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.283, "count": 34, "max_ms": 71.33, "p95_ms": 70.639}, "yolo11_640_fire_11class": {"avg_ms": 46.431, "count": 34, "max_ms": 63.076, "p95_ms": 62.975}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.37, "count": 34, "max_ms": 80.572, "p95_ms": 72.064}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.224, "count": 34, "max_ms": 234.141, "p95_ms": 221.764}, "read_interval": {"avg_ms": 758.638, "count": 33, "max_ms": 18146.321, "p95_ms": 230.474}, "triton_wait": {"avg_ms": 60.405, "count": 34, "max_ms": 80.583, "p95_ms": 72.076}, "visualize": {"avg_ms": 9.93, "count": 34, "max_ms": 17.747, "p95_ms": 10.217}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.82, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 264}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6017.336, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561740c94740] error while decoding MB 8 6, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:59:14+09:00", "last_error_summary": "[h264 @ 0x561740c94740] error while decoding MB 8 6, bytestream -11", "last_exit_at": "2026-07-01T13:57:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.128, "last_started_at": "2026-07-01T13:57:26+09:00", "pid": 11665, "process": {"exists": false, "pid": 11665}, "restart_count": 61, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 241.194, "sample_count": 1239, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31718, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:59:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T13:59:14+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T13:59:14+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T13:59:14.792825+09:00", "mtime_epoch": 1782881954.7928252, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202082}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:59:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.602, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.648, "pushed": 4.648, "read": 4.648}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.357, "count": 138, "max_ms": 10.61, "p95_ms": 10.313}, "gstreamer_push": {"avg_ms": 0.85, "count": 139, "max_ms": 0.956, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.33, "count": 138, "max_ms": 51.25, "p95_ms": 42.01}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.025, "count": 138, "max_ms": 79.603, "p95_ms": 73.133}, "yolo11_640_fire_11class": {"avg_ms": 48.765, "count": 138, "max_ms": 64.981, "p95_ms": 59.977}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.045, "count": 138, "max_ms": 78.711, "p95_ms": 75.616}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.882, "count": 139, "max_ms": 242.462, "p95_ms": 234.173}, "read_interval": {"avg_ms": 216.374, "count": 139, "max_ms": 252.443, "p95_ms": 243.369}, "triton_wait": {"avg_ms": 63.126, "count": 138, "max_ms": 79.614, "p95_ms": 75.628}, "visualize": {"avg_ms": 9.629, "count": 138, "max_ms": 10.953, "p95_ms": 10.665}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 264}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6016.516, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T13:59:14+09:00", "last_error_summary": "[h264 @ 0x561740c94740] error while decoding MB 8 6, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:59:14+09:00", "last_error_summary": "[h264 @ 0x561740c94740] error while decoding MB 8 6, bytestream -11", "last_exit_at": "2026-07-01T13:59:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.305, "last_started_at": "2026-07-01T13:57:26+09:00", "pid": 11665, "process": {"exists": false, "pid": 11665}, "restart_count": 61, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 241.194, "sample_count": 1239, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31719, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:59:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T13:59:15+09:00", "last_frame_read_age_sec": 0.226, "last_frame_read_at": "2026-07-01T13:59:14+09:00", "mjpeg": {"age_sec": 0.215, "exists": true, "mtime": "2026-07-01T13:59:14.792825+09:00", "mtime_epoch": 1782881954.7928252, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202082}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:59:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.602, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.66, "pushed": 4.66, "read": 4.626}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.36, "count": 139, "max_ms": 10.61, "p95_ms": 10.313}, "gstreamer_push": {"avg_ms": 0.85, "count": 139, "max_ms": 0.956, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.306, "count": 139, "max_ms": 51.25, "p95_ms": 42.01}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.011, "count": 139, "max_ms": 79.603, "p95_ms": 73.133}, "yolo11_640_fire_11class": {"avg_ms": 48.696, "count": 139, "max_ms": 64.981, "p95_ms": 59.977}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.018, "count": 139, "max_ms": 78.711, "p95_ms": 75.616}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.886, "count": 139, "max_ms": 242.462, "p95_ms": 234.173}, "read_interval": {"avg_ms": 216.287, "count": 138, "max_ms": 252.443, "p95_ms": 243.369}, "triton_wait": {"avg_ms": 63.099, "count": 139, "max_ms": 79.614, "p95_ms": 75.628}, "visualize": {"avg_ms": 9.633, "count": 139, "max_ms": 10.953, "p95_ms": 10.665}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.831, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 264}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6016.508, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:59:14+09:00", "last_error_summary": "[h264 @ 0x561740c94740] error while decoding MB 8 6, bytestream -11", "last_exit_at": "2026-07-01T13:59:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.308, "last_started_at": "2026-07-01T13:57:26+09:00", "pid": 11665, "process": {"exists": false, "pid": 11665}, "restart_count": 61, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 241.194, "sample_count": 1239, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31719, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:59:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T13:59:15+09:00", "last_frame_read_age_sec": 0.229, "last_frame_read_at": "2026-07-01T13:59:14+09:00", "mjpeg": {"age_sec": 0.218, "exists": true, "mtime": "2026-07-01T13:59:14.792825+09:00", "mtime_epoch": 1782881954.7928252, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202082}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:59:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.602, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.659, "pushed": 4.659, "read": 4.626}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.36, "count": 139, "max_ms": 10.61, "p95_ms": 10.313}, "gstreamer_push": {"avg_ms": 0.85, "count": 139, "max_ms": 0.956, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.306, "count": 139, "max_ms": 51.25, "p95_ms": 42.01}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.011, "count": 139, "max_ms": 79.603, "p95_ms": 73.133}, "yolo11_640_fire_11class": {"avg_ms": 48.696, "count": 139, "max_ms": 64.981, "p95_ms": 59.977}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.018, "count": 139, "max_ms": 78.711, "p95_ms": 75.616}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.886, "count": 139, "max_ms": 242.462, "p95_ms": 234.173}, "read_interval": {"avg_ms": 216.287, "count": 138, "max_ms": 252.443, "p95_ms": 243.369}, "triton_wait": {"avg_ms": 63.099, "count": 139, "max_ms": 79.614, "p95_ms": 75.628}, "visualize": {"avg_ms": 9.633, "count": 139, "max_ms": 10.953, "p95_ms": 10.665}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.834, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 264}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6117.348, "limit_mb": 8796093022207.996}, "detail": {"pid": 11832, "restart_count": 62, "start_count": 63}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T13:59:14+09:00", "last_error_summary": "[h264 @ 0x561740c94740] error while decoding MB 8 6, bytestream -11", "last_exit_at": "2026-07-01T13:59:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T13:59:17+09:00", "pid": 11832, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.207, "name": "ffmpeg", "pid": 11832, "rss_mb": 139.207, "state": "S (sleeping)", "threads": 106}, "restart_count": 62, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 241.228, "sample_count": 1216, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31719, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T13:59:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.367, "last_frame_pushed_at": "2026-07-01T13:59:15+09:00", "last_frame_read_age_sec": 4.588, "last_frame_read_at": "2026-07-01T13:59:14+09:00", "mjpeg": {"age_sec": 4.349, "exists": true, "mtime": "2026-07-01T13:59:15.020829+09:00", "mtime_epoch": 1782881955.0208294, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194486}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T13:59:19+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.602, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.986, "pushed": 3.986, "read": 3.952}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.354, "count": 119, "max_ms": 10.61, "p95_ms": 10.353}, "gstreamer_push": {"avg_ms": 0.847, "count": 119, "max_ms": 0.895, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.258, "count": 119, "max_ms": 51.25, "p95_ms": 42.01}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.516, "count": 119, "max_ms": 79.603, "p95_ms": 73.133}, "yolo11_640_fire_11class": {"avg_ms": 48.911, "count": 119, "max_ms": 64.981, "p95_ms": 60.015}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.352, "count": 119, "max_ms": 78.711, "p95_ms": 75.357}}, "truncated_count": 0}, "processing_total": {"avg_ms": 206.519, "count": 119, "max_ms": 239.934, "p95_ms": 233.901}, "read_interval": {"avg_ms": 215.84, "count": 118, "max_ms": 249.121, "p95_ms": 243.179}, "triton_wait": {"avg_ms": 63.445, "count": 119, "max_ms": 79.614, "p95_ms": 75.369}, "visualize": {"avg_ms": 9.615, "count": 119, "max_ms": 10.953, "p95_ms": 10.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.855, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 269}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5973.066, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55839f5253c0] error while decoding MB 18 10, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:01:23+09:00", "last_error_summary": "[h264 @ 0x55839f5253c0] error while decoding MB 18 10, bytestream -5", "last_exit_at": "2026-07-01T13:59:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.782, "last_started_at": "2026-07-01T13:59:17+09:00", "pid": 11832, "process": {"exists": false, "pid": 11832}, "restart_count": 62, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.664, "sample_count": 1252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32308, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:01:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T14:01:23+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T14:01:23+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T14:01:23.107163+09:00", "mtime_epoch": 1782882083.1071634, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200619}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:01:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.859, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.973, "pushed": 4.973, "read": 4.973}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.709, "count": 148, "max_ms": 10.545, "p95_ms": 9.934}, "gstreamer_push": {"avg_ms": 0.851, "count": 149, "max_ms": 0.925, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.772, "count": 148, "max_ms": 47.277, "p95_ms": 40.545}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.7, "count": 148, "max_ms": 75.794, "p95_ms": 71.664}, "yolo11_640_fire_11class": {"avg_ms": 47.634, "count": 148, "max_ms": 61.176, "p95_ms": 57.786}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.578, "count": 148, "max_ms": 80.113, "p95_ms": 74.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.461, "count": 149, "max_ms": 236.826, "p95_ms": 217.186}, "read_interval": {"avg_ms": 202.06, "count": 149, "max_ms": 247.667, "p95_ms": 226.527}, "triton_wait": {"avg_ms": 61.741, "count": 148, "max_ms": 80.126, "p95_ms": 74.476}, "visualize": {"avg_ms": 9.079, "count": 148, "max_ms": 11.313, "p95_ms": 10.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 269}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5972.625, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:01:23+09:00", "last_error_summary": "[h264 @ 0x55839f5253c0] error while decoding MB 18 10, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:01:23+09:00", "last_error_summary": "[h264 @ 0x55839f5253c0] error while decoding MB 18 10, bytestream -5", "last_exit_at": "2026-07-01T14:01:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.922, "last_started_at": "2026-07-01T13:59:17+09:00", "pid": 11832, "process": {"exists": false, "pid": 11832}, "restart_count": 62, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.664, "sample_count": 1253, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32309, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:01:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:01:23+09:00", "last_frame_read_age_sec": 0.192, "last_frame_read_at": "2026-07-01T14:01:23+09:00", "mjpeg": {"age_sec": 0.184, "exists": true, "mtime": "2026-07-01T14:01:23.107163+09:00", "mtime_epoch": 1782882083.1071634, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200619}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:01:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.859, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.985, "pushed": 4.985, "read": 4.952}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.711, "count": 148, "max_ms": 10.545, "p95_ms": 9.934}, "gstreamer_push": {"avg_ms": 0.852, "count": 149, "max_ms": 0.925, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.833, "count": 148, "max_ms": 47.277, "p95_ms": 40.545}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.718, "count": 148, "max_ms": 75.794, "p95_ms": 71.664}, "yolo11_640_fire_11class": {"avg_ms": 47.649, "count": 148, "max_ms": 61.176, "p95_ms": 57.786}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.603, "count": 148, "max_ms": 80.113, "p95_ms": 74.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.379, "count": 149, "max_ms": 236.826, "p95_ms": 217.186}, "read_interval": {"avg_ms": 202.016, "count": 148, "max_ms": 247.667, "p95_ms": 226.527}, "triton_wait": {"avg_ms": 61.767, "count": 148, "max_ms": 80.126, "p95_ms": 74.476}, "visualize": {"avg_ms": 9.072, "count": 148, "max_ms": 11.313, "p95_ms": 10.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 269}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5972.648, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:01:23+09:00", "last_error_summary": "[h264 @ 0x55839f5253c0] error while decoding MB 18 10, bytestream -5", "last_exit_at": "2026-07-01T14:01:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.926, "last_started_at": "2026-07-01T13:59:17+09:00", "pid": 11832, "process": {"exists": false, "pid": 11832}, "restart_count": 62, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.664, "sample_count": 1253, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32309, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:01:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T14:01:23+09:00", "last_frame_read_age_sec": 0.196, "last_frame_read_at": "2026-07-01T14:01:23+09:00", "mjpeg": {"age_sec": 0.188, "exists": true, "mtime": "2026-07-01T14:01:23.107163+09:00", "mtime_epoch": 1782882083.1071634, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200619}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:01:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.859, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.984, "pushed": 4.984, "read": 4.951}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.711, "count": 148, "max_ms": 10.545, "p95_ms": 9.934}, "gstreamer_push": {"avg_ms": 0.852, "count": 149, "max_ms": 0.925, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.833, "count": 148, "max_ms": 47.277, "p95_ms": 40.545}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.718, "count": 148, "max_ms": 75.794, "p95_ms": 71.664}, "yolo11_640_fire_11class": {"avg_ms": 47.649, "count": 148, "max_ms": 61.176, "p95_ms": 57.786}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.603, "count": 148, "max_ms": 80.113, "p95_ms": 74.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.379, "count": 149, "max_ms": 236.826, "p95_ms": 217.186}, "read_interval": {"avg_ms": 202.016, "count": 148, "max_ms": 247.667, "p95_ms": 226.527}, "triton_wait": {"avg_ms": 61.767, "count": 148, "max_ms": 80.126, "p95_ms": 74.476}, "visualize": {"avg_ms": 9.072, "count": 148, "max_ms": 11.313, "p95_ms": 10.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 269}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6077.852, "limit_mb": 8796093022207.996}, "detail": {"pid": 11939, "restart_count": 63, "start_count": 64}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:01:23+09:00", "last_error_summary": "[h264 @ 0x55839f5253c0] error while decoding MB 18 10, bytestream -5", "last_exit_at": "2026-07-01T14:01:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:01:26+09:00", "pid": 11939, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.414, "name": "ffmpeg", "pid": 11939, "rss_mb": 136.414, "state": "S (sleeping)", "threads": 106}, "restart_count": 63, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.161, "sample_count": 1233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32309, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:01:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.745, "last_frame_pushed_at": "2026-07-01T14:01:23+09:00", "last_frame_read_age_sec": 4.935, "last_frame_read_at": "2026-07-01T14:01:23+09:00", "mjpeg": {"age_sec": 4.727, "exists": true, "mtime": "2026-07-01T14:01:23.307167+09:00", "mtime_epoch": 1782882083.3071673, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200695}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:01:28+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2365.859, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.234, "pushed": 4.234, "read": 4.2}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.7, "count": 126, "max_ms": 10.545, "p95_ms": 9.939}, "gstreamer_push": {"avg_ms": 0.852, "count": 127, "max_ms": 0.925, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.846, "count": 126, "max_ms": 47.277, "p95_ms": 40.067}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.887, "count": 126, "max_ms": 74.466, "p95_ms": 69.856}, "yolo11_640_fire_11class": {"avg_ms": 48.054, "count": 126, "max_ms": 61.176, "p95_ms": 58.694}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.921, "count": 126, "max_ms": 80.113, "p95_ms": 74.454}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.823, "count": 127, "max_ms": 236.826, "p95_ms": 217.186}, "read_interval": {"avg_ms": 200.484, "count": 126, "max_ms": 247.667, "p95_ms": 226.639}, "triton_wait": {"avg_ms": 61.994, "count": 126, "max_ms": 80.126, "p95_ms": 74.476}, "visualize": {"avg_ms": 8.996, "count": 126, "max_ms": 11.313, "p95_ms": 10.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 274}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5974.637, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c72d6d5580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c72d6d5580] error while decoding MB 0 4, bytestream 54 [h264 @ 0x55c72d713180] error while decoding MB 51 8, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:03:34+09:00", "last_error_summary": "[h264 @ 0x55c72d6d5580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c72d6d5580] error while decoding MB 0 4, bytestream 54 [h264 @ 0x55c72d713180] error while decoding MB 51 8, bytestream -5", "last_exit_at": "2026-07-01T14:01:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.244, "last_started_at": "2026-07-01T14:01:26+09:00", "pid": 11939, "process": {"exists": false, "pid": 11939}, "restart_count": 63, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.257, "sample_count": 1384, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32916, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:03:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.052, "last_frame_pushed_at": "2026-07-01T14:03:34+09:00", "last_frame_read_age_sec": 0.044, "last_frame_read_at": "2026-07-01T14:03:34+09:00", "mjpeg": {"age_sec": 0.035, "exists": true, "mtime": "2026-07-01T14:03:34.241548+09:00", "mtime_epoch": 1782882214.2415478, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197854}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:03:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2366.891, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.96, "pushed": 4.96, "read": 4.96}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.853, "count": 148, "max_ms": 12.522, "p95_ms": 10.647}, "gstreamer_push": {"avg_ms": 0.861, "count": 148, "max_ms": 1.129, "p95_ms": 0.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.42, "count": 148, "max_ms": 48.717, "p95_ms": 42.521}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.75, "count": 148, "max_ms": 78.483, "p95_ms": 71.242}, "yolo11_640_fire_11class": {"avg_ms": 47.627, "count": 148, "max_ms": 62.178, "p95_ms": 58.397}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.591, "count": 148, "max_ms": 77.276, "p95_ms": 74.053}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.964, "count": 148, "max_ms": 247.7, "p95_ms": 221.814}, "read_interval": {"avg_ms": 202.775, "count": 148, "max_ms": 259.232, "p95_ms": 232.005}, "triton_wait": {"avg_ms": 61.733, "count": 148, "max_ms": 78.494, "p95_ms": 74.7}, "visualize": {"avg_ms": 9.14, "count": 148, "max_ms": 12.815, "p95_ms": 11.815}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.838, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 274}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5973.793, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:03:34+09:00", "last_error_summary": "[h264 @ 0x55c72d6d5580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c72d6d5580] error while decoding MB 0 4, bytestream 54 [h264 @ 0x55c72d713180] error while decoding MB 51 8, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:03:34+09:00", "last_error_summary": "[h264 @ 0x55c72d6d5580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c72d6d5580] error while decoding MB 0 4, bytestream 54 [h264 @ 0x55c72d713180] error while decoding MB 51 8, bytestream -5", "last_exit_at": "2026-07-01T14:03:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.442, "last_started_at": "2026-07-01T14:01:26+09:00", "pid": 11939, "process": {"exists": false, "pid": 11939}, "restart_count": 63, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.257, "sample_count": 1384, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32917, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:03:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T14:03:34+09:00", "last_frame_read_age_sec": 0.242, "last_frame_read_at": "2026-07-01T14:03:34+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-01T14:03:34.241548+09:00", "mtime_epoch": 1782882214.2415478, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197854}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:03:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2366.891, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.964, "pushed": 4.964, "read": 4.93}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.848, "count": 148, "max_ms": 12.522, "p95_ms": 10.647}, "gstreamer_push": {"avg_ms": 0.861, "count": 148, "max_ms": 1.129, "p95_ms": 0.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.43, "count": 148, "max_ms": 48.717, "p95_ms": 42.521}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.887, "count": 148, "max_ms": 78.483, "p95_ms": 71.244}, "yolo11_640_fire_11class": {"avg_ms": 47.769, "count": 148, "max_ms": 62.178, "p95_ms": 58.832}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.688, "count": 148, "max_ms": 77.276, "p95_ms": 74.053}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.095, "count": 148, "max_ms": 247.7, "p95_ms": 226.76}, "read_interval": {"avg_ms": 202.616, "count": 147, "max_ms": 259.232, "p95_ms": 232.005}, "triton_wait": {"avg_ms": 61.836, "count": 148, "max_ms": 78.494, "p95_ms": 74.7}, "visualize": {"avg_ms": 9.13, "count": 148, "max_ms": 12.815, "p95_ms": 11.716}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.815, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 274}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5974.16, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:03:34+09:00", "last_error_summary": "[h264 @ 0x55c72d6d5580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c72d6d5580] error while decoding MB 0 4, bytestream 54 [h264 @ 0x55c72d713180] error while decoding MB 51 8, bytestream -5", "last_exit_at": "2026-07-01T14:03:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.445, "last_started_at": "2026-07-01T14:01:26+09:00", "pid": 11939, "process": {"exists": false, "pid": 11939}, "restart_count": 63, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.257, "sample_count": 1384, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32917, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:03:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T14:03:34+09:00", "last_frame_read_age_sec": 0.246, "last_frame_read_at": "2026-07-01T14:03:34+09:00", "mjpeg": {"age_sec": 0.237, "exists": true, "mtime": "2026-07-01T14:03:34.241548+09:00", "mtime_epoch": 1782882214.2415478, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197854}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:03:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2366.891, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.963, "pushed": 4.963, "read": 4.93}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.848, "count": 148, "max_ms": 12.522, "p95_ms": 10.647}, "gstreamer_push": {"avg_ms": 0.861, "count": 148, "max_ms": 1.129, "p95_ms": 0.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.43, "count": 148, "max_ms": 48.717, "p95_ms": 42.521}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.887, "count": 148, "max_ms": 78.483, "p95_ms": 71.244}, "yolo11_640_fire_11class": {"avg_ms": 47.769, "count": 148, "max_ms": 62.178, "p95_ms": 58.832}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.688, "count": 148, "max_ms": 77.276, "p95_ms": 74.053}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.095, "count": 148, "max_ms": 247.7, "p95_ms": 226.76}, "read_interval": {"avg_ms": 202.616, "count": 147, "max_ms": 259.232, "p95_ms": 232.005}, "triton_wait": {"avg_ms": 61.836, "count": 148, "max_ms": 78.494, "p95_ms": 74.7}, "visualize": {"avg_ms": 9.13, "count": 148, "max_ms": 12.815, "p95_ms": 11.716}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.819, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 274}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6079.496, "limit_mb": 8796093022207.996}, "detail": {"pid": 12046, "restart_count": 64, "start_count": 65}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:03:34+09:00", "last_error_summary": "[h264 @ 0x55c72d6d5580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c72d6d5580] error while decoding MB 0 4, bytestream 54 [h264 @ 0x55c72d713180] error while decoding MB 51 8, bytestream -5", "last_exit_at": "2026-07-01T14:03:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:03:36+09:00", "pid": 12046, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.766, "name": "ffmpeg", "pid": 12046, "rss_mb": 136.766, "state": "S (sleeping)", "threads": 106}, "restart_count": 64, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.581, "sample_count": 1364, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32917, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:03:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.479, "last_frame_pushed_at": "2026-07-01T14:03:34+09:00", "last_frame_read_age_sec": 4.717, "last_frame_read_at": "2026-07-01T14:03:34+09:00", "mjpeg": {"age_sec": 4.464, "exists": true, "mtime": "2026-07-01T14:03:34.485552+09:00", "mtime_epoch": 1782882214.4855523, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198291}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:03:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2366.891, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.254, "pushed": 4.254, "read": 4.221}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.786, "count": 127, "max_ms": 12.522, "p95_ms": 10.384}, "gstreamer_push": {"avg_ms": 0.861, "count": 127, "max_ms": 1.129, "p95_ms": 0.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.364, "count": 127, "max_ms": 48.717, "p95_ms": 42.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.563, "count": 127, "max_ms": 75.208, "p95_ms": 71.158}, "yolo11_640_fire_11class": {"avg_ms": 47.412, "count": 127, "max_ms": 61.738, "p95_ms": 58.397}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.42, "count": 127, "max_ms": 77.276, "p95_ms": 74.376}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.411, "count": 127, "max_ms": 244.599, "p95_ms": 219.075}, "read_interval": {"avg_ms": 200.866, "count": 126, "max_ms": 254.035, "p95_ms": 228.455}, "triton_wait": {"avg_ms": 61.502, "count": 127, "max_ms": 77.287, "p95_ms": 74.7}, "visualize": {"avg_ms": 8.956, "count": 127, "max_ms": 11.128, "p95_ms": 10.438}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.853, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 279}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5975.098, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:03:34+09:00", "last_error_summary": "[h264 @ 0x55c72d6d5580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c72d6d5580] error while decoding MB 0 4, bytestream 54 [h264 @ 0x55c72d713180] error while decoding MB 51 8, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:03:34+09:00", "last_error_summary": "[h264 @ 0x55c72d6d5580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c72d6d5580] error while decoding MB 0 4, bytestream 54 [h264 @ 0x55c72d713180] error while decoding MB 51 8, bytestream -5", "last_exit_at": "2026-07-01T14:05:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.522, "last_started_at": "2026-07-01T14:03:36+09:00", "pid": 12046, "process": {"exists": false, "pid": 12046}, "restart_count": 64, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.026, "sample_count": 1425, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33524, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:05:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:05:40+09:00", "last_frame_read_age_sec": 0.187, "last_frame_read_at": "2026-07-01T14:05:40+09:00", "mjpeg": {"age_sec": 0.178, "exists": true, "mtime": "2026-07-01T14:05:40.291836+09:00", "mtime_epoch": 1782882340.2918358, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200052}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:05:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.148, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 154, "pushed_frames": 154, "read_frames": 153, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.137, "pushed": 5.137, "read": 5.104}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.479, "count": 153, "max_ms": 10.96, "p95_ms": 9.728}, "gstreamer_push": {"avg_ms": 0.86, "count": 154, "max_ms": 1.101, "p95_ms": 0.965}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.842, "count": 153, "max_ms": 47.013, "p95_ms": 44.852}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.35, "count": 153, "max_ms": 74.777, "p95_ms": 70.397}, "yolo11_640_fire_11class": {"avg_ms": 47.274, "count": 153, "max_ms": 61.176, "p95_ms": 58.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.611, "count": 153, "max_ms": 80.537, "p95_ms": 73.668}}, "truncated_count": 0}, "processing_total": {"avg_ms": 186.172, "count": 154, "max_ms": 219.045, "p95_ms": 215.066}, "read_interval": {"avg_ms": 196.105, "count": 153, "max_ms": 228.583, "p95_ms": 225.047}, "triton_wait": {"avg_ms": 61.662, "count": 153, "max_ms": 80.549, "p95_ms": 74.039}, "visualize": {"avg_ms": 8.792, "count": 153, "max_ms": 10.848, "p95_ms": 10.289}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 279}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5975.09, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:03:34+09:00", "last_error_summary": "[h264 @ 0x55c72d6d5580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c72d6d5580] error while decoding MB 0 4, bytestream 54 [h264 @ 0x55c72d713180] error while decoding MB 51 8, bytestream -5", "last_exit_at": "2026-07-01T14:05:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.525, "last_started_at": "2026-07-01T14:03:36+09:00", "pid": 12046, "process": {"exists": false, "pid": 12046}, "restart_count": 64, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.026, "sample_count": 1425, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33524, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:05:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:05:40+09:00", "last_frame_read_age_sec": 0.191, "last_frame_read_at": "2026-07-01T14:05:40+09:00", "mjpeg": {"age_sec": 0.182, "exists": true, "mtime": "2026-07-01T14:05:40.291836+09:00", "mtime_epoch": 1782882340.2918358, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200052}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:05:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.148, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 154, "pushed_frames": 154, "read_frames": 153, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.136, "pushed": 5.136, "read": 5.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.479, "count": 153, "max_ms": 10.96, "p95_ms": 9.728}, "gstreamer_push": {"avg_ms": 0.86, "count": 154, "max_ms": 1.101, "p95_ms": 0.965}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.842, "count": 153, "max_ms": 47.013, "p95_ms": 44.852}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.35, "count": 153, "max_ms": 74.777, "p95_ms": 70.397}, "yolo11_640_fire_11class": {"avg_ms": 47.274, "count": 153, "max_ms": 61.176, "p95_ms": 58.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.611, "count": 153, "max_ms": 80.537, "p95_ms": 73.668}}, "truncated_count": 0}, "processing_total": {"avg_ms": 186.172, "count": 154, "max_ms": 219.045, "p95_ms": 215.066}, "read_interval": {"avg_ms": 196.105, "count": 153, "max_ms": 228.583, "p95_ms": 225.047}, "triton_wait": {"avg_ms": 61.662, "count": 153, "max_ms": 80.549, "p95_ms": 74.039}, "visualize": {"avg_ms": 8.792, "count": 153, "max_ms": 10.848, "p95_ms": 10.289}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 279}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6083.891, "limit_mb": 8796093022207.996}, "detail": {"pid": 12153, "restart_count": 65, "start_count": 66}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:03:34+09:00", "last_error_summary": "[h264 @ 0x55c72d6d5580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c72d6d5580] error while decoding MB 0 4, bytestream 54 [h264 @ 0x55c72d713180] error while decoding MB 51 8, bytestream -5", "last_exit_at": "2026-07-01T14:05:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:05:42+09:00", "pid": 12153, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.191, "name": "ffmpeg", "pid": 12153, "rss_mb": 139.191, "state": "S (sleeping)", "threads": 106}, "restart_count": 65, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.341, "sample_count": 1405, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33524, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:05:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.142, "last_frame_pushed_at": "2026-07-01T14:05:40+09:00", "last_frame_read_age_sec": 4.326, "last_frame_read_at": "2026-07-01T14:05:40+09:00", "mjpeg": {"age_sec": 4.125, "exists": true, "mtime": "2026-07-01T14:05:40.483839+09:00", "mtime_epoch": 1782882340.4838393, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200679}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:05:44+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.148, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.439, "pushed": 4.439, "read": 4.405}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.511, "count": 132, "max_ms": 10.96, "p95_ms": 9.778}, "gstreamer_push": {"avg_ms": 0.863, "count": 133, "max_ms": 1.101, "p95_ms": 0.977}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.001, "count": 132, "max_ms": 47.013, "p95_ms": 45.217}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.491, "count": 132, "max_ms": 74.777, "p95_ms": 70.397}, "yolo11_640_fire_11class": {"avg_ms": 47.497, "count": 132, "max_ms": 61.011, "p95_ms": 58.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.871, "count": 132, "max_ms": 80.537, "p95_ms": 73.82}}, "truncated_count": 0}, "processing_total": {"avg_ms": 185.69, "count": 133, "max_ms": 219.045, "p95_ms": 215.066}, "read_interval": {"avg_ms": 195.63, "count": 132, "max_ms": 228.583, "p95_ms": 225.047}, "triton_wait": {"avg_ms": 61.923, "count": 132, "max_ms": 80.549, "p95_ms": 74.08}, "visualize": {"avg_ms": 8.779, "count": 132, "max_ms": 10.848, "p95_ms": 10.342}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 284}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5977.691, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c229060f00] error while decoding MB 4 4, bytestream -20", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:07:46+09:00", "last_error_summary": "[h264 @ 0x55c229060f00] error while decoding MB 4 4, bytestream -20", "last_exit_at": "2026-07-01T14:05:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.396, "last_started_at": "2026-07-01T14:05:42+09:00", "pid": 12153, "process": {"exists": false, "pid": 12153}, "restart_count": 65, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.997, "sample_count": 1445, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34122, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:07:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T14:07:45+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T14:07:45+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T14:07:45.962114+09:00", "mtime_epoch": 1782882465.9621136, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194589}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:07:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.406, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.879, "pushed": 4.879, "read": 4.879}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.999, "count": 145, "max_ms": 14.477, "p95_ms": 11.119}, "gstreamer_push": {"avg_ms": 0.857, "count": 146, "max_ms": 0.962, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.375, "count": 145, "max_ms": 47.366, "p95_ms": 39.156}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.179, "count": 145, "max_ms": 74.158, "p95_ms": 71.046}, "yolo11_640_fire_11class": {"avg_ms": 47.287, "count": 145, "max_ms": 61.86, "p95_ms": 60.688}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.272, "count": 145, "max_ms": 80.626, "p95_ms": 74.125}}, "truncated_count": 0}, "processing_total": {"avg_ms": 196.702, "count": 146, "max_ms": 243.77, "p95_ms": 234.354}, "read_interval": {"avg_ms": 205.865, "count": 146, "max_ms": 254.049, "p95_ms": 243.932}, "triton_wait": {"avg_ms": 62.341, "count": 145, "max_ms": 80.638, "p95_ms": 74.17}, "visualize": {"avg_ms": 9.298, "count": 145, "max_ms": 13.163, "p95_ms": 11.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 284}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5977.129, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:07:46+09:00", "last_error_summary": "[h264 @ 0x55c229060f00] error while decoding MB 4 4, bytestream -20", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:07:46+09:00", "last_error_summary": "[h264 @ 0x55c229060f00] error while decoding MB 4 4, bytestream -20", "last_exit_at": "2026-07-01T14:07:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.574, "last_started_at": "2026-07-01T14:05:42+09:00", "pid": 12153, "process": {"exists": false, "pid": 12153}, "restart_count": 65, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 219.045, "sample_count": 1445, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34123, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:07:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:07:46+09:00", "last_frame_read_age_sec": 0.229, "last_frame_read_at": "2026-07-01T14:07:45+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-01T14:07:45.962114+09:00", "mtime_epoch": 1782882465.9621136, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194589}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:07:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.406, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.888, "pushed": 4.888, "read": 4.855}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.032, "count": 146, "max_ms": 14.477, "p95_ms": 11.124}, "gstreamer_push": {"avg_ms": 0.857, "count": 146, "max_ms": 0.962, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.361, "count": 146, "max_ms": 47.366, "p95_ms": 39.156}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.148, "count": 146, "max_ms": 74.158, "p95_ms": 71.046}, "yolo11_640_fire_11class": {"avg_ms": 47.236, "count": 146, "max_ms": 61.86, "p95_ms": 60.688}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.229, "count": 146, "max_ms": 80.626, "p95_ms": 74.125}}, "truncated_count": 0}, "processing_total": {"avg_ms": 196.983, "count": 146, "max_ms": 243.77, "p95_ms": 234.354}, "read_interval": {"avg_ms": 205.956, "count": 145, "max_ms": 254.049, "p95_ms": 243.932}, "triton_wait": {"avg_ms": 62.297, "count": 146, "max_ms": 80.638, "p95_ms": 74.17}, "visualize": {"avg_ms": 9.312, "count": 146, "max_ms": 13.163, "p95_ms": 11.39}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 284}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5977.121, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:07:46+09:00", "last_error_summary": "[h264 @ 0x55c229060f00] error while decoding MB 4 4, bytestream -20", "last_exit_at": "2026-07-01T14:07:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.578, "last_started_at": "2026-07-01T14:05:42+09:00", "pid": 12153, "process": {"exists": false, "pid": 12153}, "restart_count": 65, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 219.045, "sample_count": 1445, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34123, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:07:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:07:46+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T14:07:45+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T14:07:45.962114+09:00", "mtime_epoch": 1782882465.9621136, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194589}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:07:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.406, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.888, "pushed": 4.888, "read": 4.854}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.032, "count": 146, "max_ms": 14.477, "p95_ms": 11.124}, "gstreamer_push": {"avg_ms": 0.857, "count": 146, "max_ms": 0.962, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.361, "count": 146, "max_ms": 47.366, "p95_ms": 39.156}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.148, "count": 146, "max_ms": 74.158, "p95_ms": 71.046}, "yolo11_640_fire_11class": {"avg_ms": 47.236, "count": 146, "max_ms": 61.86, "p95_ms": 60.688}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.229, "count": 146, "max_ms": 80.626, "p95_ms": 74.125}}, "truncated_count": 0}, "processing_total": {"avg_ms": 196.983, "count": 146, "max_ms": 243.77, "p95_ms": 234.354}, "read_interval": {"avg_ms": 205.956, "count": 145, "max_ms": 254.049, "p95_ms": 243.932}, "triton_wait": {"avg_ms": 62.297, "count": 146, "max_ms": 80.638, "p95_ms": 74.17}, "visualize": {"avg_ms": 9.312, "count": 146, "max_ms": 13.163, "p95_ms": 11.39}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 284}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6082.027, "limit_mb": 8796093022207.996}, "detail": {"pid": 12260, "restart_count": 66, "start_count": 67}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:07:46+09:00", "last_error_summary": "[h264 @ 0x55c229060f00] error while decoding MB 4 4, bytestream -20", "last_exit_at": "2026-07-01T14:07:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:07:48+09:00", "pid": 12260, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.66, "name": "ffmpeg", "pid": 12260, "rss_mb": 136.66, "state": "S (sleeping)", "threads": 106}, "restart_count": 66, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 219.064, "sample_count": 1425, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34123, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:07:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.14, "last_frame_pushed_at": "2026-07-01T14:07:46+09:00", "last_frame_read_age_sec": 4.366, "last_frame_read_at": "2026-07-01T14:07:45+09:00", "mjpeg": {"age_sec": 4.125, "exists": true, "mtime": "2026-07-01T14:07:46.194118+09:00", "mtime_epoch": 1782882466.1941178, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194903}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:07:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.406, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.169, "pushed": 4.169, "read": 4.136}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.177, "count": 124, "max_ms": 14.477, "p95_ms": 11.18}, "gstreamer_push": {"avg_ms": 0.858, "count": 125, "max_ms": 0.962, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.406, "count": 124, "max_ms": 47.366, "p95_ms": 38.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.328, "count": 124, "max_ms": 74.158, "p95_ms": 71.355}, "yolo11_640_fire_11class": {"avg_ms": 47.528, "count": 124, "max_ms": 61.86, "p95_ms": 60.806}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.485, "count": 124, "max_ms": 80.626, "p95_ms": 74.125}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.954, "count": 125, "max_ms": 243.77, "p95_ms": 234.381}, "read_interval": {"avg_ms": 208.006, "count": 124, "max_ms": 254.049, "p95_ms": 244.188}, "triton_wait": {"avg_ms": 62.563, "count": 124, "max_ms": 80.638, "p95_ms": 74.17}, "visualize": {"avg_ms": 9.426, "count": 124, "max_ms": 13.163, "p95_ms": 11.571}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5978.566, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55de02795940] error while decoding MB 65 4, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:09:53+09:00", "last_error_summary": "[h264 @ 0x55de02795940] error while decoding MB 65 4, bytestream -5", "last_exit_at": "2026-07-01T14:07:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.209, "last_started_at": "2026-07-01T14:07:48+09:00", "pid": 12260, "process": {"exists": false, "pid": 12260}, "restart_count": 66, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.267, "sample_count": 1436, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34722, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:09:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T14:09:53+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T14:09:53+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T14:09:53.480422+09:00", "mtime_epoch": 1782882593.4804225, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193969}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:09:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.664, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 150, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.021, "pushed": 5.021, "read": 5.021}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.513, "count": 150, "max_ms": 11.712, "p95_ms": 9.465}, "gstreamer_push": {"avg_ms": 0.856, "count": 150, "max_ms": 0.947, "p95_ms": 0.896}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.965, "count": 150, "max_ms": 48.715, "p95_ms": 44.0}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.887, "count": 150, "max_ms": 76.255, "p95_ms": 72.676}, "yolo11_640_fire_11class": {"avg_ms": 48.308, "count": 150, "max_ms": 64.544, "p95_ms": 59.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.25, "count": 150, "max_ms": 79.696, "p95_ms": 74.667}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.086, "count": 150, "max_ms": 223.184, "p95_ms": 212.903}, "read_interval": {"avg_ms": 199.937, "count": 150, "max_ms": 232.701, "p95_ms": 222.357}, "triton_wait": {"avg_ms": 62.379, "count": 150, "max_ms": 79.708, "p95_ms": 74.68}, "visualize": {"avg_ms": 8.894, "count": 150, "max_ms": 11.706, "p95_ms": 9.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.872, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5977.906, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:09:53+09:00", "last_error_summary": "[h264 @ 0x55de02795940] error while decoding MB 65 4, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:09:53+09:00", "last_error_summary": "[h264 @ 0x55de02795940] error while decoding MB 65 4, bytestream -5", "last_exit_at": "2026-07-01T14:09:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.384, "last_started_at": "2026-07-01T14:07:48+09:00", "pid": 12260, "process": {"exists": false, "pid": 12260}, "restart_count": 66, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.495, "sample_count": 1436, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34723, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:09:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:09:53+09:00", "last_frame_read_age_sec": 0.228, "last_frame_read_at": "2026-07-01T14:09:53+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T14:09:53.480422+09:00", "mtime_epoch": 1782882593.4804225, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193969}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:09:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.664, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.024, "pushed": 5.024, "read": 4.991}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.54, "count": 150, "max_ms": 11.712, "p95_ms": 9.466}, "gstreamer_push": {"avg_ms": 0.856, "count": 150, "max_ms": 0.947, "p95_ms": 0.896}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.972, "count": 150, "max_ms": 48.715, "p95_ms": 44.0}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.873, "count": 150, "max_ms": 76.255, "p95_ms": 72.676}, "yolo11_640_fire_11class": {"avg_ms": 48.283, "count": 150, "max_ms": 64.544, "p95_ms": 59.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.251, "count": 150, "max_ms": 79.696, "p95_ms": 74.667}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.433, "count": 150, "max_ms": 223.607, "p95_ms": 213.526}, "read_interval": {"avg_ms": 200.055, "count": 149, "max_ms": 232.701, "p95_ms": 222.357}, "triton_wait": {"avg_ms": 62.379, "count": 150, "max_ms": 79.708, "p95_ms": 74.68}, "visualize": {"avg_ms": 8.918, "count": 150, "max_ms": 11.706, "p95_ms": 9.93}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.855, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5977.898, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:09:53+09:00", "last_error_summary": "[h264 @ 0x55de02795940] error while decoding MB 65 4, bytestream -5", "last_exit_at": "2026-07-01T14:09:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.388, "last_started_at": "2026-07-01T14:07:48+09:00", "pid": 12260, "process": {"exists": false, "pid": 12260}, "restart_count": 66, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.495, "sample_count": 1436, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34723, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:09:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:09:53+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T14:09:53+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-01T14:09:53.480422+09:00", "mtime_epoch": 1782882593.4804225, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193969}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:09:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.664, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.024, "pushed": 5.024, "read": 4.99}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.54, "count": 150, "max_ms": 11.712, "p95_ms": 9.466}, "gstreamer_push": {"avg_ms": 0.856, "count": 150, "max_ms": 0.947, "p95_ms": 0.896}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.972, "count": 150, "max_ms": 48.715, "p95_ms": 44.0}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.873, "count": 150, "max_ms": 76.255, "p95_ms": 72.676}, "yolo11_640_fire_11class": {"avg_ms": 48.283, "count": 150, "max_ms": 64.544, "p95_ms": 59.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.251, "count": 150, "max_ms": 79.696, "p95_ms": 74.667}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.433, "count": 150, "max_ms": 223.607, "p95_ms": 213.526}, "read_interval": {"avg_ms": 200.055, "count": 149, "max_ms": 232.701, "p95_ms": 222.357}, "triton_wait": {"avg_ms": 62.379, "count": 150, "max_ms": 79.708, "p95_ms": 74.68}, "visualize": {"avg_ms": 8.918, "count": 150, "max_ms": 11.706, "p95_ms": 9.93}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.858, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6084.051, "limit_mb": 8796093022207.996}, "detail": {"pid": 12367, "restart_count": 67, "start_count": 68}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:09:53+09:00", "last_error_summary": "[h264 @ 0x55de02795940] error while decoding MB 65 4, bytestream -5", "last_exit_at": "2026-07-01T14:09:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:09:55+09:00", "pid": 12367, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.867, "name": "ffmpeg", "pid": 12367, "rss_mb": 136.867, "state": "S (sleeping)", "threads": 106}, "restart_count": 67, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.826, "sample_count": 1417, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34723, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:09:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.097, "last_frame_pushed_at": "2026-07-01T14:09:53+09:00", "last_frame_read_age_sec": 4.322, "last_frame_read_at": "2026-07-01T14:09:53+09:00", "mjpeg": {"age_sec": 4.088, "exists": true, "mtime": "2026-07-01T14:09:53.708427+09:00", "mtime_epoch": 1782882593.7084267, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185800}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:09:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.664, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.315, "pushed": 4.315, "read": 4.282}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.599, "count": 129, "max_ms": 11.712, "p95_ms": 9.622}, "gstreamer_push": {"avg_ms": 0.856, "count": 129, "max_ms": 0.947, "p95_ms": 0.895}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.892, "count": 129, "max_ms": 47.47, "p95_ms": 43.878}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.077, "count": 129, "max_ms": 76.255, "p95_ms": 72.705}, "yolo11_640_fire_11class": {"avg_ms": 48.474, "count": 129, "max_ms": 64.544, "p95_ms": 59.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.577, "count": 129, "max_ms": 79.696, "p95_ms": 74.67}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.702, "count": 129, "max_ms": 223.607, "p95_ms": 215.208}, "read_interval": {"avg_ms": 201.297, "count": 128, "max_ms": 232.701, "p95_ms": 223.132}, "triton_wait": {"avg_ms": 62.713, "count": 129, "max_ms": 79.708, "p95_ms": 74.681}, "visualize": {"avg_ms": 8.95, "count": 129, "max_ms": 11.706, "p95_ms": 9.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/18/tracking_analysis_20260701_141144_b50cdfe7.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "18", "cgroup_memory": {"current_mb": 6318.391, "limit_mb": 8796093022207.996}, "detail": {"pid": 12476, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:11:44+09:00", "pid": 12476, "process": {"exists": true, "fd_count": 5, "hwm_mb": 125.312, "name": "ffmpeg", "pid": 12476, "rss_mb": 125.312, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.939}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b50cdfe7", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 767678.248, "exists": true, "mtime": "2026-06-22T16:57:08.753328+09:00", "mtime_epoch": 1782115028.7533283, "path": "/data/playlist/streaming/18/mjpeg.jpg", "size_bytes": 274492}, "monitor_url": "http://192.168.50.244:1223/mjpeg/18", "observed_at": "2026-07-01T14:11:47+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 704.277, "name": "celery", "pid": 239, "rss_mb": 676.641, "state": "R (running)", "threads": 118}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "file:///data/playlist/uploads/260430_steam_테스트용/260430_2509서초서리풀_1분짜리샘플영상들_720_10fps편집본/CAM1__segment_290-350s__preprocessed.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:11:44+09:00", "state": "running", "task_id": "c3f29df6-f3ba-4749-a43c-229d64c56f74", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 294}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6233.98, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e5bc1a5180] error while decoding MB 10 37, bytestream -34", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:11:57+09:00", "last_error_summary": "[h264 @ 0x55e5bc1a5180] error while decoding MB 10 37, bytestream -34", "last_exit_at": "2026-07-01T14:09:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.127, "last_started_at": "2026-07-01T14:09:55+09:00", "pid": 12367, "process": {"exists": false, "pid": 12367}, "restart_count": 67, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.739, "sample_count": 1433, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35316, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:11:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T14:11:57+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-01T14:11:57+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T14:11:57.882673+09:00", "mtime_epoch": 1782882717.8826733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:11:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.922, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 145, "pushed_frames": 145, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.837, "pushed": 4.837, "read": 4.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.981, "count": 144, "max_ms": 11.575, "p95_ms": 10.16}, "gstreamer_push": {"avg_ms": 0.874, "count": 145, "max_ms": 0.974, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.112, "count": 144, "max_ms": 47.674, "p95_ms": 43.49}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.579, "count": 144, "max_ms": 93.998, "p95_ms": 79.618}, "yolo11_640_fire_11class": {"avg_ms": 48.059, "count": 144, "max_ms": 65.25, "p95_ms": 59.254}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.393, "count": 144, "max_ms": 106.657, "p95_ms": 86.942}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.833, "count": 145, "max_ms": 254.073, "p95_ms": 227.349}, "read_interval": {"avg_ms": 207.819, "count": 145, "max_ms": 263.921, "p95_ms": 236.778}, "triton_wait": {"avg_ms": 63.659, "count": 144, "max_ms": 106.67, "p95_ms": 87.08}, "visualize": {"avg_ms": 9.331, "count": 144, "max_ms": 11.474, "p95_ms": 10.872}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 294}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6232.715, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:11:57+09:00", "last_error_summary": "[h264 @ 0x55e5bc1a5180] error while decoding MB 10 37, bytestream -34", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:11:57+09:00", "last_error_summary": "[h264 @ 0x55e5bc1a5180] error while decoding MB 10 37, bytestream -34", "last_exit_at": "2026-07-01T14:11:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.284, "last_started_at": "2026-07-01T14:09:55+09:00", "pid": 12367, "process": {"exists": false, "pid": 12367}, "restart_count": 67, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.739, "sample_count": 1433, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35317, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:11:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:11:58+09:00", "last_frame_read_age_sec": 0.206, "last_frame_read_at": "2026-07-01T14:11:57+09:00", "mjpeg": {"age_sec": 0.197, "exists": true, "mtime": "2026-07-01T14:11:57.882673+09:00", "mtime_epoch": 1782882717.8826733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:11:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.922, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 145, "pushed_frames": 145, "read_frames": 144, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.843, "pushed": 4.843, "read": 4.809}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.986, "count": 144, "max_ms": 11.575, "p95_ms": 10.16}, "gstreamer_push": {"avg_ms": 0.874, "count": 145, "max_ms": 0.974, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.124, "count": 144, "max_ms": 47.674, "p95_ms": 43.49}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.554, "count": 144, "max_ms": 93.998, "p95_ms": 79.618}, "yolo11_640_fire_11class": {"avg_ms": 48.044, "count": 144, "max_ms": 65.25, "p95_ms": 59.254}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.375, "count": 144, "max_ms": 106.657, "p95_ms": 86.942}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.825, "count": 145, "max_ms": 254.073, "p95_ms": 227.349}, "read_interval": {"avg_ms": 207.792, "count": 144, "max_ms": 263.921, "p95_ms": 236.778}, "triton_wait": {"avg_ms": 63.641, "count": 144, "max_ms": 106.67, "p95_ms": 87.08}, "visualize": {"avg_ms": 9.336, "count": 144, "max_ms": 11.474, "p95_ms": 10.872}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 294}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6232.797, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:11:57+09:00", "last_error_summary": "[h264 @ 0x55e5bc1a5180] error while decoding MB 10 37, bytestream -34", "last_exit_at": "2026-07-01T14:11:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.288, "last_started_at": "2026-07-01T14:09:55+09:00", "pid": 12367, "process": {"exists": false, "pid": 12367}, "restart_count": 67, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.739, "sample_count": 1433, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35317, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:11:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:11:58+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T14:11:57+09:00", "mjpeg": {"age_sec": 0.201, "exists": true, "mtime": "2026-07-01T14:11:57.882673+09:00", "mtime_epoch": 1782882717.8826733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189482}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:11:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.922, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 145, "pushed_frames": 145, "read_frames": 144, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.842, "pushed": 4.842, "read": 4.809}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.986, "count": 144, "max_ms": 11.575, "p95_ms": 10.16}, "gstreamer_push": {"avg_ms": 0.874, "count": 145, "max_ms": 0.974, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.124, "count": 144, "max_ms": 47.674, "p95_ms": 43.49}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.554, "count": 144, "max_ms": 93.998, "p95_ms": 79.618}, "yolo11_640_fire_11class": {"avg_ms": 48.044, "count": 144, "max_ms": 65.25, "p95_ms": 59.254}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.375, "count": 144, "max_ms": 106.657, "p95_ms": 86.942}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.825, "count": 145, "max_ms": 254.073, "p95_ms": 227.349}, "read_interval": {"avg_ms": 207.792, "count": 144, "max_ms": 263.921, "p95_ms": 236.778}, "triton_wait": {"avg_ms": 63.641, "count": 144, "max_ms": 106.67, "p95_ms": 87.08}, "visualize": {"avg_ms": 9.336, "count": 144, "max_ms": 11.474, "p95_ms": 10.872}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 294}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6344.438, "limit_mb": 8796093022207.996}, "detail": {"pid": 12584, "restart_count": 68, "start_count": 69}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:11:57+09:00", "last_error_summary": "[h264 @ 0x55e5bc1a5180] error while decoding MB 10 37, bytestream -34", "last_exit_at": "2026-07-01T14:12:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:12:00+09:00", "pid": 12584, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.762, "name": "ffmpeg", "pid": 12584, "rss_mb": 141.762, "state": "S (sleeping)", "threads": 106}, "restart_count": 68, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.152, "sample_count": 1411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35317, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:11:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.513, "last_frame_pushed_at": "2026-07-01T14:11:58+09:00", "last_frame_read_age_sec": 4.715, "last_frame_read_at": "2026-07-01T14:11:57+09:00", "mjpeg": {"age_sec": 4.495, "exists": true, "mtime": "2026-07-01T14:11:58.094677+09:00", "mtime_epoch": 1782882718.094677, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189588}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:12:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2367.922, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.135, "pushed": 4.135, "read": 4.102}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.998, "count": 123, "max_ms": 11.575, "p95_ms": 10.178}, "gstreamer_push": {"avg_ms": 0.874, "count": 124, "max_ms": 0.974, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 34.261, "count": 123, "max_ms": 47.674, "p95_ms": 43.49}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.827, "count": 123, "max_ms": 93.998, "p95_ms": 80.743}, "yolo11_640_fire_11class": {"avg_ms": 47.901, "count": 123, "max_ms": 63.915, "p95_ms": 58.092}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.72, "count": 123, "max_ms": 106.657, "p95_ms": 87.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.426, "count": 124, "max_ms": 254.073, "p95_ms": 225.91}, "read_interval": {"avg_ms": 207.4, "count": 123, "max_ms": 263.921, "p95_ms": 235.948}, "triton_wait": {"avg_ms": 64.012, "count": 123, "max_ms": 106.67, "p95_ms": 87.991}, "visualize": {"avg_ms": 9.257, "count": 123, "max_ms": 11.316, "p95_ms": 10.756}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 299}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5981.941, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ec69f23000] error while decoding MB 36 14, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:14:04+09:00", "last_error_summary": "[h264 @ 0x55ec69f23000] error while decoding MB 36 14, bytestream -5", "last_exit_at": "2026-07-01T14:12:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.762, "last_started_at": "2026-07-01T14:12:00+09:00", "pid": 12584, "process": {"exists": false, "pid": 12584}, "restart_count": 68, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.749, "sample_count": 1427, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35902, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:14:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T14:14:04+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T14:14:04+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T14:14:04.308959+09:00", "mtime_epoch": 1782882844.308959, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204726}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:14:04+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.18, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.883, "pushed": 4.883, "read": 4.883}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.927, "count": 145, "max_ms": 14.407, "p95_ms": 10.101}, "gstreamer_push": {"avg_ms": 0.864, "count": 146, "max_ms": 1.008, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.616, "count": 145, "max_ms": 47.288, "p95_ms": 43.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.406, "count": 145, "max_ms": 78.647, "p95_ms": 72.622}, "yolo11_640_fire_11class": {"avg_ms": 47.551, "count": 145, "max_ms": 65.166, "p95_ms": 60.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.684, "count": 145, "max_ms": 82.872, "p95_ms": 73.914}}, "truncated_count": 0}, "processing_total": {"avg_ms": 196.129, "count": 146, "max_ms": 240.877, "p95_ms": 222.041}, "read_interval": {"avg_ms": 205.722, "count": 146, "max_ms": 250.318, "p95_ms": 231.692}, "triton_wait": {"avg_ms": 61.819, "count": 145, "max_ms": 82.885, "p95_ms": 75.158}, "visualize": {"avg_ms": 9.312, "count": 145, "max_ms": 13.316, "p95_ms": 11.606}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 299}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5981.262, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:14:04+09:00", "last_error_summary": "[h264 @ 0x55ec69f23000] error while decoding MB 36 14, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:14:04+09:00", "last_error_summary": "[h264 @ 0x55ec69f23000] error while decoding MB 36 14, bytestream -5", "last_exit_at": "2026-07-01T14:14:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.923, "last_started_at": "2026-07-01T14:12:00+09:00", "pid": 12584, "process": {"exists": false, "pid": 12584}, "restart_count": 68, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.749, "sample_count": 1427, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:14:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:14:04+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T14:14:04+09:00", "mjpeg": {"age_sec": 0.202, "exists": true, "mtime": "2026-07-01T14:14:04.308959+09:00", "mtime_epoch": 1782882844.308959, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204726}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:14:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.889, "pushed": 4.889, "read": 4.855}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.926, "count": 146, "max_ms": 14.407, "p95_ms": 10.101}, "gstreamer_push": {"avg_ms": 0.864, "count": 146, "max_ms": 1.008, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.616, "count": 146, "max_ms": 47.288, "p95_ms": 43.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.431, "count": 146, "max_ms": 78.647, "p95_ms": 72.622}, "yolo11_640_fire_11class": {"avg_ms": 47.547, "count": 146, "max_ms": 65.166, "p95_ms": 60.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.693, "count": 146, "max_ms": 82.872, "p95_ms": 73.914}}, "truncated_count": 0}, "processing_total": {"avg_ms": 196.266, "count": 146, "max_ms": 240.877, "p95_ms": 222.041}, "read_interval": {"avg_ms": 205.796, "count": 145, "max_ms": 250.318, "p95_ms": 231.692}, "triton_wait": {"avg_ms": 61.828, "count": 146, "max_ms": 82.885, "p95_ms": 75.158}, "visualize": {"avg_ms": 9.314, "count": 146, "max_ms": 13.316, "p95_ms": 11.606}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.863, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 299}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5981.254, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:14:04+09:00", "last_error_summary": "[h264 @ 0x55ec69f23000] error while decoding MB 36 14, bytestream -5", "last_exit_at": "2026-07-01T14:14:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.926, "last_started_at": "2026-07-01T14:12:00+09:00", "pid": 12584, "process": {"exists": false, "pid": 12584}, "restart_count": 68, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.749, "sample_count": 1427, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:14:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:14:04+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T14:14:04+09:00", "mjpeg": {"age_sec": 0.206, "exists": true, "mtime": "2026-07-01T14:14:04.308959+09:00", "mtime_epoch": 1782882844.308959, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204726}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:14:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.888, "pushed": 4.888, "read": 4.855}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.926, "count": 146, "max_ms": 14.407, "p95_ms": 10.101}, "gstreamer_push": {"avg_ms": 0.864, "count": 146, "max_ms": 1.008, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.616, "count": 146, "max_ms": 47.288, "p95_ms": 43.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.431, "count": 146, "max_ms": 78.647, "p95_ms": 72.622}, "yolo11_640_fire_11class": {"avg_ms": 47.547, "count": 146, "max_ms": 65.166, "p95_ms": 60.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.693, "count": 146, "max_ms": 82.872, "p95_ms": 73.914}}, "truncated_count": 0}, "processing_total": {"avg_ms": 196.266, "count": 146, "max_ms": 240.877, "p95_ms": 222.041}, "read_interval": {"avg_ms": 205.796, "count": 145, "max_ms": 250.318, "p95_ms": 231.692}, "triton_wait": {"avg_ms": 61.828, "count": 146, "max_ms": 82.885, "p95_ms": 75.158}, "visualize": {"avg_ms": 9.314, "count": 146, "max_ms": 13.316, "p95_ms": 11.606}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 299}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6083.742, "limit_mb": 8796093022207.996}, "detail": {"pid": 12691, "restart_count": 69, "start_count": 70}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:14:04+09:00", "last_error_summary": "[h264 @ 0x55ec69f23000] error while decoding MB 36 14, bytestream -5", "last_exit_at": "2026-07-01T14:14:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:14:07+09:00", "pid": 12691, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.555, "name": "ffmpeg", "pid": 12691, "rss_mb": 133.555, "state": "S (sleeping)", "threads": 106}, "restart_count": 69, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.773, "sample_count": 1402, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:14:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.723, "last_frame_pushed_at": "2026-07-01T14:14:04+09:00", "last_frame_read_age_sec": 4.93, "last_frame_read_at": "2026-07-01T14:14:04+09:00", "mjpeg": {"age_sec": 4.71, "exists": true, "mtime": "2026-07-01T14:14:04.520963+09:00", "mtime_epoch": 1782882844.5209627, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204761}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:14:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.18, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.115, "pushed": 4.115, "read": 4.081}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.015, "count": 122, "max_ms": 14.407, "p95_ms": 10.119}, "gstreamer_push": {"avg_ms": 0.865, "count": 123, "max_ms": 1.008, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.398, "count": 122, "max_ms": 47.122, "p95_ms": 43.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.558, "count": 122, "max_ms": 78.566, "p95_ms": 72.612}, "yolo11_640_fire_11class": {"avg_ms": 47.411, "count": 122, "max_ms": 65.166, "p95_ms": 59.259}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.75, "count": 122, "max_ms": 82.872, "p95_ms": 73.914}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.061, "count": 123, "max_ms": 240.877, "p95_ms": 222.348}, "read_interval": {"avg_ms": 206.596, "count": 122, "max_ms": 250.318, "p95_ms": 233.06}, "triton_wait": {"avg_ms": 61.852, "count": 122, "max_ms": 82.885, "p95_ms": 74.975}, "visualize": {"avg_ms": 9.332, "count": 122, "max_ms": 13.316, "p95_ms": 11.858}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 304}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5983.258, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560bba9321c0] error while decoding MB 53 7, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:16:12+09:00", "last_error_summary": "[h264 @ 0x560bba9321c0] error while decoding MB 53 7, bytestream -11", "last_exit_at": "2026-07-01T14:14:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.315, "last_started_at": "2026-07-01T14:14:07+09:00", "pid": 12691, "process": {"exists": false, "pid": 12691}, "restart_count": 69, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.564, "sample_count": 1391, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36487, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:16:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T14:16:12+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T14:16:12+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T14:16:12.503276+09:00", "mtime_epoch": 1782882972.5032756, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191812}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:16:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.18, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 144, "pushed_frames": 144, "read_frames": 144, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.818, "pushed": 4.818, "read": 4.818}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.067, "count": 143, "max_ms": 14.241, "p95_ms": 10.407}, "gstreamer_push": {"avg_ms": 0.867, "count": 144, "max_ms": 1.11, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.368, "count": 143, "max_ms": 58.095, "p95_ms": 43.372}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.445, "count": 143, "max_ms": 73.898, "p95_ms": 69.039}, "yolo11_640_fire_11class": {"avg_ms": 46.962, "count": 143, "max_ms": 60.795, "p95_ms": 56.861}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.849, "count": 143, "max_ms": 75.701, "p95_ms": 72.316}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.902, "count": 144, "max_ms": 248.247, "p95_ms": 225.107}, "read_interval": {"avg_ms": 208.582, "count": 144, "max_ms": 259.501, "p95_ms": 235.273}, "triton_wait": {"avg_ms": 61.063, "count": 143, "max_ms": 75.713, "p95_ms": 72.412}, "visualize": {"avg_ms": 9.446, "count": 143, "max_ms": 14.91, "p95_ms": 10.995}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 304}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5982.84, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:16:12+09:00", "last_error_summary": "[h264 @ 0x560bba9321c0] error while decoding MB 53 7, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:16:12+09:00", "last_error_summary": "[h264 @ 0x560bba9321c0] error while decoding MB 53 7, bytestream -11", "last_exit_at": "2026-07-01T14:16:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.471, "last_started_at": "2026-07-01T14:14:07+09:00", "pid": 12691, "process": {"exists": false, "pid": 12691}, "restart_count": 69, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.564, "sample_count": 1391, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36488, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:16:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:16:12+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T14:16:12+09:00", "mjpeg": {"age_sec": 0.198, "exists": true, "mtime": "2026-07-01T14:16:12.503276+09:00", "mtime_epoch": 1782882972.5032756, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191812}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:16:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 144, "pushed_frames": 144, "read_frames": 143, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.827, "pushed": 4.827, "read": 4.794}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.067, "count": 144, "max_ms": 14.241, "p95_ms": 10.407}, "gstreamer_push": {"avg_ms": 0.866, "count": 144, "max_ms": 1.11, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.376, "count": 144, "max_ms": 58.095, "p95_ms": 43.372}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.464, "count": 144, "max_ms": 73.898, "p95_ms": 69.039}, "yolo11_640_fire_11class": {"avg_ms": 46.978, "count": 144, "max_ms": 60.795, "p95_ms": 56.861}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.867, "count": 144, "max_ms": 75.701, "p95_ms": 72.316}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.909, "count": 144, "max_ms": 248.247, "p95_ms": 225.107}, "read_interval": {"avg_ms": 208.54, "count": 143, "max_ms": 259.501, "p95_ms": 235.273}, "triton_wait": {"avg_ms": 61.08, "count": 144, "max_ms": 75.713, "p95_ms": 72.412}, "visualize": {"avg_ms": 9.449, "count": 144, "max_ms": 14.91, "p95_ms": 10.995}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.83, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 304}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5982.91, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:16:12+09:00", "last_error_summary": "[h264 @ 0x560bba9321c0] error while decoding MB 53 7, bytestream -11", "last_exit_at": "2026-07-01T14:16:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.475, "last_started_at": "2026-07-01T14:14:07+09:00", "pid": 12691, "process": {"exists": false, "pid": 12691}, "restart_count": 69, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.564, "sample_count": 1391, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36488, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:16:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:16:12+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T14:16:12+09:00", "mjpeg": {"age_sec": 0.202, "exists": true, "mtime": "2026-07-01T14:16:12.503276+09:00", "mtime_epoch": 1782882972.5032756, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191812}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:16:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 144, "pushed_frames": 144, "read_frames": 143, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.827, "pushed": 4.827, "read": 4.793}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.067, "count": 144, "max_ms": 14.241, "p95_ms": 10.407}, "gstreamer_push": {"avg_ms": 0.866, "count": 144, "max_ms": 1.11, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.376, "count": 144, "max_ms": 58.095, "p95_ms": 43.372}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.464, "count": 144, "max_ms": 73.898, "p95_ms": 69.039}, "yolo11_640_fire_11class": {"avg_ms": 46.978, "count": 144, "max_ms": 60.795, "p95_ms": 56.861}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.867, "count": 144, "max_ms": 75.701, "p95_ms": 72.316}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.909, "count": 144, "max_ms": 248.247, "p95_ms": 225.107}, "read_interval": {"avg_ms": 208.54, "count": 143, "max_ms": 259.501, "p95_ms": 235.273}, "triton_wait": {"avg_ms": 61.08, "count": 144, "max_ms": 75.713, "p95_ms": 72.412}, "visualize": {"avg_ms": 9.449, "count": 144, "max_ms": 14.91, "p95_ms": 10.995}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.833, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 304}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6085.754, "limit_mb": 8796093022207.996}, "detail": {"pid": 12798, "restart_count": 70, "start_count": 71}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:16:12+09:00", "last_error_summary": "[h264 @ 0x560bba9321c0] error while decoding MB 53 7, bytestream -11", "last_exit_at": "2026-07-01T14:16:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:16:15+09:00", "pid": 12798, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.969, "name": "ffmpeg", "pid": 12798, "rss_mb": 133.969, "state": "S (sleeping)", "threads": 106}, "restart_count": 70, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.634, "sample_count": 1369, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36488, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:16:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.542, "last_frame_pushed_at": "2026-07-01T14:16:12+09:00", "last_frame_read_age_sec": 4.749, "last_frame_read_at": "2026-07-01T14:16:12+09:00", "mjpeg": {"age_sec": 4.525, "exists": true, "mtime": "2026-07-01T14:16:12.715279+09:00", "mtime_epoch": 1782882972.7152793, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185519}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:16:17+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.18, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.114, "pushed": 4.114, "read": 4.081}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.015, "count": 122, "max_ms": 14.241, "p95_ms": 10.681}, "gstreamer_push": {"avg_ms": 0.867, "count": 123, "max_ms": 1.11, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.605, "count": 122, "max_ms": 58.095, "p95_ms": 44.468}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.776, "count": 122, "max_ms": 72.667, "p95_ms": 68.949}, "yolo11_640_fire_11class": {"avg_ms": 46.965, "count": 122, "max_ms": 59.423, "p95_ms": 56.861}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.922, "count": 122, "max_ms": 75.701, "p95_ms": 72.131}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.136, "count": 123, "max_ms": 248.247, "p95_ms": 225.446}, "read_interval": {"avg_ms": 207.733, "count": 122, "max_ms": 259.501, "p95_ms": 235.349}, "triton_wait": {"avg_ms": 61.161, "count": 122, "max_ms": 75.713, "p95_ms": 72.244}, "visualize": {"avg_ms": 9.424, "count": 122, "max_ms": 14.91, "p95_ms": 10.995}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 309}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5984.52, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561e9e5b5c00] error while decoding MB 73 14, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:18:28+09:00", "last_error_summary": "[h264 @ 0x561e9e5b5c00] error while decoding MB 73 14, bytestream -12", "last_exit_at": "2026-07-01T14:16:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.392, "last_started_at": "2026-07-01T14:16:15+09:00", "pid": 12798, "process": {"exists": false, "pid": 12798}, "restart_count": 70, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.107, "sample_count": 1380, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37108, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:18:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T14:18:28+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T14:18:28+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T14:18:28.589734+09:00", "mtime_epoch": 1782883108.5897338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:18:28+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.695, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.701, "pushed": 4.701, "read": 4.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.212, "count": 140, "max_ms": 10.236, "p95_ms": 9.758}, "gstreamer_push": {"avg_ms": 0.857, "count": 141, "max_ms": 0.923, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.309, "count": 140, "max_ms": 48.279, "p95_ms": 43.064}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.043, "count": 140, "max_ms": 75.369, "p95_ms": 73.321}, "yolo11_640_fire_11class": {"avg_ms": 47.49, "count": 140, "max_ms": 62.952, "p95_ms": 59.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.528, "count": 140, "max_ms": 79.396, "p95_ms": 73.939}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.099, "count": 141, "max_ms": 225.38, "p95_ms": 219.874}, "read_interval": {"avg_ms": 213.954, "count": 141, "max_ms": 235.195, "p95_ms": 229.403}, "triton_wait": {"avg_ms": 61.693, "count": 140, "max_ms": 79.408, "p95_ms": 74.808}, "visualize": {"avg_ms": 9.687, "count": 140, "max_ms": 11.513, "p95_ms": 10.331}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 309}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5983.93, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:18:28+09:00", "last_error_summary": "[h264 @ 0x561e9e5b5c00] error while decoding MB 73 14, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:18:28+09:00", "last_error_summary": "[h264 @ 0x561e9e5b5c00] error while decoding MB 73 14, bytestream -12", "last_exit_at": "2026-07-01T14:18:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.554, "last_started_at": "2026-07-01T14:16:15+09:00", "pid": 12798, "process": {"exists": false, "pid": 12798}, "restart_count": 70, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.107, "sample_count": 1380, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37109, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:18:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:18:28+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T14:18:28+09:00", "mjpeg": {"age_sec": 0.204, "exists": true, "mtime": "2026-07-01T14:18:28.589734+09:00", "mtime_epoch": 1782883108.5897338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:18:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.695, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.711, "pushed": 4.711, "read": 4.677}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.216, "count": 140, "max_ms": 10.236, "p95_ms": 9.879}, "gstreamer_push": {"avg_ms": 0.857, "count": 141, "max_ms": 0.923, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.308, "count": 140, "max_ms": 48.279, "p95_ms": 43.064}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.024, "count": 140, "max_ms": 75.369, "p95_ms": 73.321}, "yolo11_640_fire_11class": {"avg_ms": 47.475, "count": 140, "max_ms": 62.952, "p95_ms": 59.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.515, "count": 140, "max_ms": 79.396, "p95_ms": 73.939}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.983, "count": 141, "max_ms": 223.692, "p95_ms": 219.49}, "read_interval": {"avg_ms": 213.802, "count": 140, "max_ms": 233.306, "p95_ms": 229.034}, "triton_wait": {"avg_ms": 61.679, "count": 140, "max_ms": 79.408, "p95_ms": 74.808}, "visualize": {"avg_ms": 9.7, "count": 140, "max_ms": 12.112, "p95_ms": 10.557}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 309}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5984.047, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:18:28+09:00", "last_error_summary": "[h264 @ 0x561e9e5b5c00] error while decoding MB 73 14, bytestream -12", "last_exit_at": "2026-07-01T14:18:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.558, "last_started_at": "2026-07-01T14:16:15+09:00", "pid": 12798, "process": {"exists": false, "pid": 12798}, "restart_count": 70, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.107, "sample_count": 1380, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37109, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:18:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:18:28+09:00", "last_frame_read_age_sec": 0.217, "last_frame_read_at": "2026-07-01T14:18:28+09:00", "mjpeg": {"age_sec": 0.208, "exists": true, "mtime": "2026-07-01T14:18:28.589734+09:00", "mtime_epoch": 1782883108.5897338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:18:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.695, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.71, "pushed": 4.71, "read": 4.677}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.216, "count": 140, "max_ms": 10.236, "p95_ms": 9.879}, "gstreamer_push": {"avg_ms": 0.857, "count": 141, "max_ms": 0.923, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.308, "count": 140, "max_ms": 48.279, "p95_ms": 43.064}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.024, "count": 140, "max_ms": 75.369, "p95_ms": 73.321}, "yolo11_640_fire_11class": {"avg_ms": 47.475, "count": 140, "max_ms": 62.952, "p95_ms": 59.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.515, "count": 140, "max_ms": 79.396, "p95_ms": 73.939}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.983, "count": 141, "max_ms": 223.692, "p95_ms": 219.49}, "read_interval": {"avg_ms": 213.802, "count": 140, "max_ms": 233.306, "p95_ms": 229.034}, "triton_wait": {"avg_ms": 61.679, "count": 140, "max_ms": 79.408, "p95_ms": 74.808}, "visualize": {"avg_ms": 9.7, "count": 140, "max_ms": 12.112, "p95_ms": 10.557}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 309}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6090.035, "limit_mb": 8796093022207.996}, "detail": {"pid": 12905, "restart_count": 71, "start_count": 72}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:18:28+09:00", "last_error_summary": "[h264 @ 0x561e9e5b5c00] error while decoding MB 73 14, bytestream -12", "last_exit_at": "2026-07-01T14:18:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:18:31+09:00", "pid": 12905, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.719, "name": "ffmpeg", "pid": 12905, "rss_mb": 136.719, "state": "S (sleeping)", "threads": 106}, "restart_count": 71, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.502, "sample_count": 1356, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37109, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:18:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.944, "last_frame_pushed_at": "2026-07-01T14:18:28+09:00", "last_frame_read_age_sec": 5.154, "last_frame_read_at": "2026-07-01T14:18:28+09:00", "mjpeg": {"age_sec": 4.925, "exists": true, "mtime": "2026-07-01T14:18:28.809738+09:00", "mtime_epoch": 1782883108.8097377, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184635}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:18:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.695, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.928, "pushed": 3.928, "read": 3.894}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.234, "count": 117, "max_ms": 10.236, "p95_ms": 9.885}, "gstreamer_push": {"avg_ms": 0.858, "count": 117, "max_ms": 0.923, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.367, "count": 117, "max_ms": 48.279, "p95_ms": 43.448}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.049, "count": 117, "max_ms": 75.369, "p95_ms": 73.66}, "yolo11_640_fire_11class": {"avg_ms": 47.7, "count": 117, "max_ms": 62.952, "p95_ms": 59.855}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.779, "count": 117, "max_ms": 79.396, "p95_ms": 75.637}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.405, "count": 117, "max_ms": 223.692, "p95_ms": 220.528}, "read_interval": {"avg_ms": 214.244, "count": 116, "max_ms": 233.306, "p95_ms": 230.221}, "triton_wait": {"avg_ms": 61.953, "count": 117, "max_ms": 79.408, "p95_ms": 75.65}, "visualize": {"avg_ms": 9.704, "count": 117, "max_ms": 12.112, "p95_ms": 10.273}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.789, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5985.273, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5582b6116c40] error while decoding MB 17 41, bytestream -32", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:20:38+09:00", "last_error_summary": "[h264 @ 0x5582b6116c40] error while decoding MB 17 41, bytestream -32", "last_exit_at": "2026-07-01T14:18:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.164, "last_started_at": "2026-07-01T14:18:31+09:00", "pid": 12905, "process": {"exists": false, "pid": 12905}, "restart_count": 71, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.036, "sample_count": 1358, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37683, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:20:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T14:20:38+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T14:20:38+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T14:20:38.852086+09:00", "mtime_epoch": 1782883238.852086, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193584}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:20:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.953, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.701, "pushed": 4.701, "read": 4.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.878, "count": 140, "max_ms": 15.411, "p95_ms": 13.64}, "gstreamer_push": {"avg_ms": 0.842, "count": 141, "max_ms": 1.05, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.325, "count": 140, "max_ms": 51.5, "p95_ms": 34.804}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.975, "count": 140, "max_ms": 79.928, "p95_ms": 72.896}, "yolo11_640_fire_11class": {"avg_ms": 47.007, "count": 140, "max_ms": 63.077, "p95_ms": 58.546}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.88, "count": 140, "max_ms": 82.452, "p95_ms": 74.471}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.999, "count": 141, "max_ms": 244.87, "p95_ms": 234.398}, "read_interval": {"avg_ms": 213.756, "count": 141, "max_ms": 254.386, "p95_ms": 243.954}, "triton_wait": {"avg_ms": 61.071, "count": 140, "max_ms": 82.465, "p95_ms": 75.411}, "visualize": {"avg_ms": 9.693, "count": 140, "max_ms": 14.052, "p95_ms": 11.736}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5985.145, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:20:38+09:00", "last_error_summary": "[h264 @ 0x5582b6116c40] error while decoding MB 17 41, bytestream -32", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:20:38+09:00", "last_error_summary": "[h264 @ 0x5582b6116c40] error while decoding MB 17 41, bytestream -32", "last_exit_at": "2026-07-01T14:20:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.323, "last_started_at": "2026-07-01T14:18:31+09:00", "pid": 12905, "process": {"exists": false, "pid": 12905}, "restart_count": 71, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.036, "sample_count": 1359, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:20:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:20:39+09:00", "last_frame_read_age_sec": 0.211, "last_frame_read_at": "2026-07-01T14:20:38+09:00", "mjpeg": {"age_sec": 0.204, "exists": true, "mtime": "2026-07-01T14:20:38.852086+09:00", "mtime_epoch": 1782883238.852086, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193584}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:20:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.953, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.708, "pushed": 4.708, "read": 4.675}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.889, "count": 140, "max_ms": 15.411, "p95_ms": 13.64}, "gstreamer_push": {"avg_ms": 0.842, "count": 141, "max_ms": 1.05, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.372, "count": 140, "max_ms": 51.5, "p95_ms": 34.804}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.074, "count": 140, "max_ms": 79.928, "p95_ms": 72.896}, "yolo11_640_fire_11class": {"avg_ms": 47.079, "count": 140, "max_ms": 63.077, "p95_ms": 58.546}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.967, "count": 140, "max_ms": 82.452, "p95_ms": 74.471}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.055, "count": 141, "max_ms": 244.87, "p95_ms": 234.398}, "read_interval": {"avg_ms": 213.792, "count": 140, "max_ms": 254.386, "p95_ms": 243.954}, "triton_wait": {"avg_ms": 61.163, "count": 140, "max_ms": 82.465, "p95_ms": 75.411}, "visualize": {"avg_ms": 9.705, "count": 140, "max_ms": 14.052, "p95_ms": 11.736}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5985.137, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:20:38+09:00", "last_error_summary": "[h264 @ 0x5582b6116c40] error while decoding MB 17 41, bytestream -32", "last_exit_at": "2026-07-01T14:20:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.326, "last_started_at": "2026-07-01T14:18:31+09:00", "pid": 12905, "process": {"exists": false, "pid": 12905}, "restart_count": 71, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.036, "sample_count": 1359, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:20:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:20:39+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T14:20:38+09:00", "mjpeg": {"age_sec": 0.208, "exists": true, "mtime": "2026-07-01T14:20:38.852086+09:00", "mtime_epoch": 1782883238.852086, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193584}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:20:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.953, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.708, "pushed": 4.708, "read": 4.674}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.889, "count": 140, "max_ms": 15.411, "p95_ms": 13.64}, "gstreamer_push": {"avg_ms": 0.842, "count": 141, "max_ms": 1.05, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.372, "count": 140, "max_ms": 51.5, "p95_ms": 34.804}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.074, "count": 140, "max_ms": 79.928, "p95_ms": 72.896}, "yolo11_640_fire_11class": {"avg_ms": 47.079, "count": 140, "max_ms": 63.077, "p95_ms": 58.546}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.967, "count": 140, "max_ms": 82.452, "p95_ms": 74.471}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.055, "count": 141, "max_ms": 244.87, "p95_ms": 234.398}, "read_interval": {"avg_ms": 213.792, "count": 140, "max_ms": 254.386, "p95_ms": 243.954}, "triton_wait": {"avg_ms": 61.163, "count": 140, "max_ms": 82.465, "p95_ms": 75.411}, "visualize": {"avg_ms": 9.705, "count": 140, "max_ms": 14.052, "p95_ms": 11.736}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6090.035, "limit_mb": 8796093022207.996}, "detail": {"pid": 13012, "restart_count": 72, "start_count": 73}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:20:38+09:00", "last_error_summary": "[h264 @ 0x5582b6116c40] error while decoding MB 17 41, bytestream -32", "last_exit_at": "2026-07-01T14:20:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:20:41+09:00", "pid": 13012, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.668, "name": "ffmpeg", "pid": 13012, "rss_mb": 136.668, "state": "S (sleeping)", "threads": 106}, "restart_count": 72, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.211, "sample_count": 1335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37684, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:20:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.793, "last_frame_pushed_at": "2026-07-01T14:20:39+09:00", "last_frame_read_age_sec": 5.0, "last_frame_read_at": "2026-07-01T14:20:38+09:00", "mjpeg": {"age_sec": 4.778, "exists": true, "mtime": "2026-07-01T14:20:39.068090+09:00", "mtime_epoch": 1782883239.06809, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193621}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:20:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.953, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.942, "pushed": 3.942, "read": 3.908}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.848, "count": 117, "max_ms": 15.348, "p95_ms": 13.724}, "gstreamer_push": {"avg_ms": 0.842, "count": 118, "max_ms": 1.05, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 30.707, "count": 117, "max_ms": 51.5, "p95_ms": 36.045}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.305, "count": 117, "max_ms": 76.473, "p95_ms": 72.956}, "yolo11_640_fire_11class": {"avg_ms": 47.494, "count": 117, "max_ms": 63.077, "p95_ms": 59.326}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.386, "count": 117, "max_ms": 82.452, "p95_ms": 75.378}}, "truncated_count": 0}, "processing_total": {"avg_ms": 205.12, "count": 118, "max_ms": 244.87, "p95_ms": 235.013}, "read_interval": {"avg_ms": 214.887, "count": 117, "max_ms": 254.386, "p95_ms": 244.77}, "triton_wait": {"avg_ms": 61.562, "count": 117, "max_ms": 82.465, "p95_ms": 76.485}, "visualize": {"avg_ms": 9.727, "count": 117, "max_ms": 14.052, "p95_ms": 11.736}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 318}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5985.992, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:20:38+09:00", "last_error_summary": "[h264 @ 0x5582b6116c40] error while decoding MB 17 41, bytestream -32", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:20:38+09:00", "last_error_summary": "[h264 @ 0x5582b6116c40] error while decoding MB 17 41, bytestream -32", "last_exit_at": "2026-07-01T14:22:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.307, "last_started_at": "2026-07-01T14:20:41+09:00", "pid": 13012, "process": {"exists": false, "pid": 13012}, "restart_count": 72, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 233.919, "sample_count": 1350, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:22:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:22:49+09:00", "last_frame_read_age_sec": 0.2, "last_frame_read_at": "2026-07-01T14:22:48+09:00", "mjpeg": {"age_sec": 0.196, "exists": true, "mtime": "2026-07-01T14:22:48.955771+09:00", "mtime_epoch": 1782883368.9557712, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187895}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:22:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.953, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.952, "pushed": 4.952, "read": 4.918}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.807, "count": 147, "max_ms": 12.459, "p95_ms": 10.193}, "gstreamer_push": {"avg_ms": 0.845, "count": 148, "max_ms": 1.343, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.96, "count": 147, "max_ms": 51.974, "p95_ms": 40.441}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.039, "count": 147, "max_ms": 79.774, "p95_ms": 73.946}, "yolo11_640_fire_11class": {"avg_ms": 46.705, "count": 147, "max_ms": 68.167, "p95_ms": 57.97}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.515, "count": 147, "max_ms": 79.257, "p95_ms": 73.212}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.643, "count": 148, "max_ms": 228.786, "p95_ms": 216.507}, "read_interval": {"avg_ms": 203.34, "count": 147, "max_ms": 239.733, "p95_ms": 227.098}, "triton_wait": {"avg_ms": 61.17, "count": 147, "max_ms": 79.785, "p95_ms": 76.386}, "visualize": {"avg_ms": 9.206, "count": 147, "max_ms": 14.367, "p95_ms": 10.731}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 318}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5985.984, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:20:38+09:00", "last_error_summary": "[h264 @ 0x5582b6116c40] error while decoding MB 17 41, bytestream -32", "last_exit_at": "2026-07-01T14:22:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.31, "last_started_at": "2026-07-01T14:20:41+09:00", "pid": 13012, "process": {"exists": false, "pid": 13012}, "restart_count": 72, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 233.919, "sample_count": 1350, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:22:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:22:49+09:00", "last_frame_read_age_sec": 0.203, "last_frame_read_at": "2026-07-01T14:22:48+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T14:22:48.955771+09:00", "mtime_epoch": 1782883368.9557712, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187895}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:22:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.953, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.951, "pushed": 4.951, "read": 4.918}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.807, "count": 147, "max_ms": 12.459, "p95_ms": 10.193}, "gstreamer_push": {"avg_ms": 0.845, "count": 148, "max_ms": 1.343, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.96, "count": 147, "max_ms": 51.974, "p95_ms": 40.441}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.039, "count": 147, "max_ms": 79.774, "p95_ms": 73.946}, "yolo11_640_fire_11class": {"avg_ms": 46.705, "count": 147, "max_ms": 68.167, "p95_ms": 57.97}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.515, "count": 147, "max_ms": 79.257, "p95_ms": 73.212}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.643, "count": 148, "max_ms": 228.786, "p95_ms": 216.507}, "read_interval": {"avg_ms": 203.34, "count": 147, "max_ms": 239.733, "p95_ms": 227.098}, "triton_wait": {"avg_ms": 61.17, "count": 147, "max_ms": 79.785, "p95_ms": 76.386}, "visualize": {"avg_ms": 9.206, "count": 147, "max_ms": 14.367, "p95_ms": 10.731}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 318}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6094.715, "limit_mb": 8796093022207.996}, "detail": {"pid": 13119, "restart_count": 73, "start_count": 74}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:20:38+09:00", "last_error_summary": "[h264 @ 0x5582b6116c40] error while decoding MB 17 41, bytestream -32", "last_exit_at": "2026-07-01T14:22:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:22:51+09:00", "pid": 13119, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.797, "name": "ffmpeg", "pid": 13119, "rss_mb": 139.797, "state": "S (sleeping)", "threads": 106}, "restart_count": 73, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.207, "sample_count": 1329, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:22:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.231, "last_frame_pushed_at": "2026-07-01T14:22:49+09:00", "last_frame_read_age_sec": 4.427, "last_frame_read_at": "2026-07-01T14:22:48+09:00", "mjpeg": {"age_sec": 4.216, "exists": true, "mtime": "2026-07-01T14:22:49.163777+09:00", "mtime_epoch": 1782883369.163777, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180129}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:22:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2368.953, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.2, "pushed": 4.2, "read": 4.2}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.814, "count": 126, "max_ms": 12.459, "p95_ms": 10.441}, "gstreamer_push": {"avg_ms": 0.846, "count": 126, "max_ms": 1.343, "p95_ms": 0.891}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.793, "count": 126, "max_ms": 51.974, "p95_ms": 39.39}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.827, "count": 126, "max_ms": 79.774, "p95_ms": 73.946}, "yolo11_640_fire_11class": {"avg_ms": 46.35, "count": 126, "max_ms": 66.353, "p95_ms": 56.372}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.336, "count": 126, "max_ms": 78.285, "p95_ms": 73.212}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.861, "count": 126, "max_ms": 228.786, "p95_ms": 218.28}, "read_interval": {"avg_ms": 204.494, "count": 126, "max_ms": 239.733, "p95_ms": 227.576}, "triton_wait": {"avg_ms": 61.009, "count": 126, "max_ms": 79.785, "p95_ms": 76.386}, "visualize": {"avg_ms": 9.301, "count": 126, "max_ms": 14.367, "p95_ms": 10.747}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 323}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5987.52, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5609bf9c6e40] error while decoding MB 72 30, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:24:54+09:00", "last_error_summary": "[h264 @ 0x5609bf9c6e40] error while decoding MB 72 30, bytestream -5", "last_exit_at": "2026-07-01T14:22:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.844, "last_started_at": "2026-07-01T14:22:51+09:00", "pid": 13119, "process": {"exists": false, "pid": 13119}, "restart_count": 73, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.407, "sample_count": 1385, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38861, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:24:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T14:24:54+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T14:24:54+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T14:24:54.183304+09:00", "mtime_epoch": 1782883494.183304, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204792}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:24:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.211, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 143, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.784, "pushed": 4.784, "read": 4.784}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.195, "count": 142, "max_ms": 12.229, "p95_ms": 10.067}, "gstreamer_push": {"avg_ms": 0.848, "count": 143, "max_ms": 0.953, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.425, "count": 142, "max_ms": 46.397, "p95_ms": 39.691}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.036, "count": 142, "max_ms": 78.802, "p95_ms": 73.261}, "yolo11_640_fire_11class": {"avg_ms": 44.925, "count": 142, "max_ms": 63.421, "p95_ms": 56.109}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.969, "count": 142, "max_ms": 77.935, "p95_ms": 74.011}}, "truncated_count": 0}, "processing_total": {"avg_ms": 200.596, "count": 143, "max_ms": 234.366, "p95_ms": 221.909}, "read_interval": {"avg_ms": 210.081, "count": 143, "max_ms": 245.178, "p95_ms": 232.489}, "triton_wait": {"avg_ms": 60.378, "count": 142, "max_ms": 78.814, "p95_ms": 75.092}, "visualize": {"avg_ms": 9.577, "count": 142, "max_ms": 11.835, "p95_ms": 10.858}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 323}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5987.051, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:24:54+09:00", "last_error_summary": "[h264 @ 0x5609bf9c6e40] error while decoding MB 72 30, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:24:54+09:00", "last_error_summary": "[h264 @ 0x5609bf9c6e40] error while decoding MB 72 30, bytestream -5", "last_exit_at": "2026-07-01T14:24:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.027, "last_started_at": "2026-07-01T14:22:51+09:00", "pid": 13119, "process": {"exists": false, "pid": 13119}, "restart_count": 73, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.801, "sample_count": 1385, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38862, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:24:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:24:54+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-01T14:24:54+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T14:24:54.183304+09:00", "mtime_epoch": 1782883494.183304, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204792}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:24:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.211, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.786, "pushed": 4.786, "read": 4.753}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.226, "count": 143, "max_ms": 13.691, "p95_ms": 10.121}, "gstreamer_push": {"avg_ms": 0.848, "count": 143, "max_ms": 0.953, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.412, "count": 143, "max_ms": 46.397, "p95_ms": 39.691}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.004, "count": 143, "max_ms": 78.802, "p95_ms": 73.261}, "yolo11_640_fire_11class": {"avg_ms": 44.913, "count": 143, "max_ms": 63.421, "p95_ms": 56.109}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.94, "count": 143, "max_ms": 77.935, "p95_ms": 74.011}}, "truncated_count": 0}, "processing_total": {"avg_ms": 200.783, "count": 143, "max_ms": 234.366, "p95_ms": 223.019}, "read_interval": {"avg_ms": 210.063, "count": 142, "max_ms": 245.178, "p95_ms": 232.489}, "triton_wait": {"avg_ms": 60.346, "count": 143, "max_ms": 78.814, "p95_ms": 75.092}, "visualize": {"avg_ms": 9.588, "count": 143, "max_ms": 11.835, "p95_ms": 10.871}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 323}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5987.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:24:54+09:00", "last_error_summary": "[h264 @ 0x5609bf9c6e40] error while decoding MB 72 30, bytestream -5", "last_exit_at": "2026-07-01T14:24:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.031, "last_started_at": "2026-07-01T14:22:51+09:00", "pid": 13119, "process": {"exists": false, "pid": 13119}, "restart_count": 73, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.801, "sample_count": 1385, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38862, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:24:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:24:54+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-01T14:24:54+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-01T14:24:54.183304+09:00", "mtime_epoch": 1782883494.183304, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204792}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:24:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.211, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.786, "pushed": 4.786, "read": 4.752}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.226, "count": 143, "max_ms": 13.691, "p95_ms": 10.121}, "gstreamer_push": {"avg_ms": 0.848, "count": 143, "max_ms": 0.953, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.412, "count": 143, "max_ms": 46.397, "p95_ms": 39.691}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.004, "count": 143, "max_ms": 78.802, "p95_ms": 73.261}, "yolo11_640_fire_11class": {"avg_ms": 44.913, "count": 143, "max_ms": 63.421, "p95_ms": 56.109}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.94, "count": 143, "max_ms": 77.935, "p95_ms": 74.011}}, "truncated_count": 0}, "processing_total": {"avg_ms": 200.783, "count": 143, "max_ms": 234.366, "p95_ms": 223.019}, "read_interval": {"avg_ms": 210.063, "count": 142, "max_ms": 245.178, "p95_ms": 232.489}, "triton_wait": {"avg_ms": 60.346, "count": 143, "max_ms": 78.814, "p95_ms": 75.092}, "visualize": {"avg_ms": 9.588, "count": 143, "max_ms": 11.835, "p95_ms": 10.871}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 323}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6094.895, "limit_mb": 8796093022207.996}, "detail": {"pid": 13226, "restart_count": 74, "start_count": 75}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:24:54+09:00", "last_error_summary": "[h264 @ 0x5609bf9c6e40] error while decoding MB 72 30, bytestream -5", "last_exit_at": "2026-07-01T14:24:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:24:56+09:00", "pid": 13226, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.176, "name": "ffmpeg", "pid": 13226, "rss_mb": 139.176, "state": "S (sleeping)", "threads": 106}, "restart_count": 74, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.823, "sample_count": 1365, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38862, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:24:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.244, "last_frame_pushed_at": "2026-07-01T14:24:54+09:00", "last_frame_read_age_sec": 4.474, "last_frame_read_at": "2026-07-01T14:24:54+09:00", "mjpeg": {"age_sec": 4.228, "exists": true, "mtime": "2026-07-01T14:24:54.419311+09:00", "mtime_epoch": 1782883494.4193106, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198363}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:24:58+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.211, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.084, "pushed": 4.084, "read": 4.051}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.28, "count": 122, "max_ms": 13.691, "p95_ms": 10.335}, "gstreamer_push": {"avg_ms": 0.849, "count": 122, "max_ms": 0.919, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.503, "count": 122, "max_ms": 46.397, "p95_ms": 39.691}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.944, "count": 122, "max_ms": 74.877, "p95_ms": 72.912}, "yolo11_640_fire_11class": {"avg_ms": 44.992, "count": 122, "max_ms": 63.421, "p95_ms": 55.949}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.127, "count": 122, "max_ms": 77.935, "p95_ms": 75.08}}, "truncated_count": 0}, "processing_total": {"avg_ms": 202.423, "count": 122, "max_ms": 234.366, "p95_ms": 223.351}, "read_interval": {"avg_ms": 211.65, "count": 121, "max_ms": 245.178, "p95_ms": 232.489}, "triton_wait": {"avg_ms": 60.491, "count": 122, "max_ms": 77.947, "p95_ms": 75.092}, "visualize": {"avg_ms": 9.674, "count": 122, "max_ms": 11.835, "p95_ms": 10.871}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 328}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5988.117, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:24:54+09:00", "last_error_summary": "[h264 @ 0x5609bf9c6e40] error while decoding MB 72 30, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:24:54+09:00", "last_error_summary": "[h264 @ 0x5609bf9c6e40] error while decoding MB 72 30, bytestream -5", "last_exit_at": "2026-07-01T14:27:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.294, "last_started_at": "2026-07-01T14:24:56+09:00", "pid": 13226, "process": {"exists": false, "pid": 13226}, "restart_count": 74, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.516, "sample_count": 1402, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39454, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:27:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:27:03+09:00", "last_frame_read_age_sec": 0.201, "last_frame_read_at": "2026-07-01T14:27:03+09:00", "mjpeg": {"age_sec": 0.19, "exists": true, "mtime": "2026-07-01T14:27:03.750712+09:00", "mtime_epoch": 1782883623.7507122, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196607}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:27:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.469, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.722, "pushed": 4.722, "read": 4.689}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.7, "count": 141, "max_ms": 16.918, "p95_ms": 13.578}, "gstreamer_push": {"avg_ms": 0.867, "count": 141, "max_ms": 1.523, "p95_ms": 0.919}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.893, "count": 141, "max_ms": 51.421, "p95_ms": 42.562}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.334, "count": 141, "max_ms": 77.473, "p95_ms": 72.603}, "yolo11_640_fire_11class": {"avg_ms": 47.883, "count": 141, "max_ms": 65.542, "p95_ms": 58.03}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.106, "count": 141, "max_ms": 79.495, "p95_ms": 75.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.524, "count": 141, "max_ms": 234.606, "p95_ms": 225.09}, "read_interval": {"avg_ms": 213.385, "count": 140, "max_ms": 245.932, "p95_ms": 234.484}, "triton_wait": {"avg_ms": 61.37, "count": 141, "max_ms": 79.507, "p95_ms": 75.934}, "visualize": {"avg_ms": 9.725, "count": 141, "max_ms": 12.778, "p95_ms": 11.146}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.857, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 328}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5988.109, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:24:54+09:00", "last_error_summary": "[h264 @ 0x5609bf9c6e40] error while decoding MB 72 30, bytestream -5", "last_exit_at": "2026-07-01T14:27:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.298, "last_started_at": "2026-07-01T14:24:56+09:00", "pid": 13226, "process": {"exists": false, "pid": 13226}, "restart_count": 74, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.516, "sample_count": 1402, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39454, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:27:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:27:03+09:00", "last_frame_read_age_sec": 0.205, "last_frame_read_at": "2026-07-01T14:27:03+09:00", "mjpeg": {"age_sec": 0.194, "exists": true, "mtime": "2026-07-01T14:27:03.750712+09:00", "mtime_epoch": 1782883623.7507122, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196607}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:27:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.469, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.722, "pushed": 4.722, "read": 4.688}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.7, "count": 141, "max_ms": 16.918, "p95_ms": 13.578}, "gstreamer_push": {"avg_ms": 0.867, "count": 141, "max_ms": 1.523, "p95_ms": 0.919}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.893, "count": 141, "max_ms": 51.421, "p95_ms": 42.562}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.334, "count": 141, "max_ms": 77.473, "p95_ms": 72.603}, "yolo11_640_fire_11class": {"avg_ms": 47.883, "count": 141, "max_ms": 65.542, "p95_ms": 58.03}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.106, "count": 141, "max_ms": 79.495, "p95_ms": 75.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.524, "count": 141, "max_ms": 234.606, "p95_ms": 225.09}, "read_interval": {"avg_ms": 213.385, "count": 140, "max_ms": 245.932, "p95_ms": 234.484}, "triton_wait": {"avg_ms": 61.37, "count": 141, "max_ms": 79.507, "p95_ms": 75.934}, "visualize": {"avg_ms": 9.725, "count": 141, "max_ms": 12.778, "p95_ms": 11.146}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.861, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 328}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6098.992, "limit_mb": 8796093022207.996}, "detail": {"pid": 13333, "restart_count": 75, "start_count": 76}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:24:54+09:00", "last_error_summary": "[h264 @ 0x5609bf9c6e40] error while decoding MB 72 30, bytestream -5", "last_exit_at": "2026-07-01T14:27:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:27:06+09:00", "pid": 13333, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.125, "name": "ffmpeg", "pid": 13333, "rss_mb": 142.125, "state": "S (sleeping)", "threads": 106}, "restart_count": 75, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.585, "sample_count": 1380, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39454, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:27:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.585, "last_frame_pushed_at": "2026-07-01T14:27:03+09:00", "last_frame_read_age_sec": 4.783, "last_frame_read_at": "2026-07-01T14:27:03+09:00", "mjpeg": {"age_sec": 4.568, "exists": true, "mtime": "2026-07-01T14:27:03.954717+09:00", "mtime_epoch": 1782883623.9547172, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197902}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:27:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.469, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.024, "pushed": 4.024, "read": 3.991}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.559, "count": 120, "max_ms": 14.842, "p95_ms": 11.211}, "gstreamer_push": {"avg_ms": 0.858, "count": 120, "max_ms": 1.223, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.864, "count": 120, "max_ms": 51.421, "p95_ms": 41.918}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.977, "count": 120, "max_ms": 77.473, "p95_ms": 72.332}, "yolo11_640_fire_11class": {"avg_ms": 47.856, "count": 120, "max_ms": 65.542, "p95_ms": 58.03}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.886, "count": 120, "max_ms": 79.018, "p95_ms": 75.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 202.411, "count": 120, "max_ms": 234.606, "p95_ms": 224.568}, "read_interval": {"avg_ms": 212.198, "count": 119, "max_ms": 244.062, "p95_ms": 234.218}, "triton_wait": {"avg_ms": 61.087, "count": 120, "max_ms": 79.03, "p95_ms": 75.934}, "visualize": {"avg_ms": 9.697, "count": 120, "max_ms": 12.778, "p95_ms": 11.059}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.821, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 333}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5989.34, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55899bd9ddc0] error while decoding MB 12 37, bytestream -36", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:29:14+09:00", "last_error_summary": "[h264 @ 0x55899bd9ddc0] error while decoding MB 12 37, bytestream -36", "last_exit_at": "2026-07-01T14:27:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.506, "last_started_at": "2026-07-01T14:27:06+09:00", "pid": 13333, "process": {"exists": false, "pid": 13333}, "restart_count": 75, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.924, "sample_count": 1364, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40033, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:29:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T14:29:13+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T14:29:13+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T14:29:13.981940+09:00", "mtime_epoch": 1782883753.9819403, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204070}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:29:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.727, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.266, "pushed": 4.266, "read": 4.266}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.551, "count": 127, "max_ms": 13.765, "p95_ms": 11.765}, "gstreamer_push": {"avg_ms": 0.859, "count": 127, "max_ms": 0.976, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.479, "count": 127, "max_ms": 47.084, "p95_ms": 41.99}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.498, "count": 127, "max_ms": 76.195, "p95_ms": 72.813}, "yolo11_640_fire_11class": {"avg_ms": 46.599, "count": 127, "max_ms": 62.87, "p95_ms": 58.361}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.129, "count": 127, "max_ms": 81.194, "p95_ms": 73.787}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.166, "count": 127, "max_ms": 267.665, "p95_ms": 256.449}, "read_interval": {"avg_ms": 236.037, "count": 127, "max_ms": 277.64, "p95_ms": 266.239}, "triton_wait": {"avg_ms": 61.267, "count": 127, "max_ms": 81.207, "p95_ms": 74.815}, "visualize": {"avg_ms": 10.86, "count": 127, "max_ms": 18.37, "p95_ms": 12.222}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.767, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 333}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5988.812, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:29:14+09:00", "last_error_summary": "[h264 @ 0x55899bd9ddc0] error while decoding MB 12 37, bytestream -36", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:29:14+09:00", "last_error_summary": "[h264 @ 0x55899bd9ddc0] error while decoding MB 12 37, bytestream -36", "last_exit_at": "2026-07-01T14:29:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.668, "last_started_at": "2026-07-01T14:27:06+09:00", "pid": 13333, "process": {"exists": false, "pid": 13333}, "restart_count": 75, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.924, "sample_count": 1364, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40034, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:29:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:29:14+09:00", "last_frame_read_age_sec": 0.215, "last_frame_read_at": "2026-07-01T14:29:13+09:00", "mjpeg": {"age_sec": 0.208, "exists": true, "mtime": "2026-07-01T14:29:13.981940+09:00", "mtime_epoch": 1782883753.9819403, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204070}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:29:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.727, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.277, "pushed": 4.277, "read": 4.243}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.568, "count": 127, "max_ms": 13.765, "p95_ms": 11.836}, "gstreamer_push": {"avg_ms": 0.86, "count": 128, "max_ms": 0.976, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.38, "count": 127, "max_ms": 47.084, "p95_ms": 41.659}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.368, "count": 127, "max_ms": 76.195, "p95_ms": 72.813}, "yolo11_640_fire_11class": {"avg_ms": 46.472, "count": 127, "max_ms": 62.87, "p95_ms": 57.868}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.945, "count": 127, "max_ms": 79.537, "p95_ms": 73.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.041, "count": 128, "max_ms": 267.665, "p95_ms": 256.449}, "read_interval": {"avg_ms": 236.037, "count": 127, "max_ms": 277.64, "p95_ms": 266.239}, "triton_wait": {"avg_ms": 61.083, "count": 127, "max_ms": 79.549, "p95_ms": 74.193}, "visualize": {"avg_ms": 10.846, "count": 127, "max_ms": 18.37, "p95_ms": 12.222}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 333}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5988.805, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:29:14+09:00", "last_error_summary": "[h264 @ 0x55899bd9ddc0] error while decoding MB 12 37, bytestream -36", "last_exit_at": "2026-07-01T14:29:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.672, "last_started_at": "2026-07-01T14:27:06+09:00", "pid": 13333, "process": {"exists": false, "pid": 13333}, "restart_count": 75, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.924, "sample_count": 1364, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40034, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:29:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:29:14+09:00", "last_frame_read_age_sec": 0.218, "last_frame_read_at": "2026-07-01T14:29:13+09:00", "mjpeg": {"age_sec": 0.211, "exists": true, "mtime": "2026-07-01T14:29:13.981940+09:00", "mtime_epoch": 1782883753.9819403, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204070}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:29:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.727, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.276, "pushed": 4.276, "read": 4.243}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.568, "count": 127, "max_ms": 13.765, "p95_ms": 11.836}, "gstreamer_push": {"avg_ms": 0.86, "count": 128, "max_ms": 0.976, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.38, "count": 127, "max_ms": 47.084, "p95_ms": 41.659}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.368, "count": 127, "max_ms": 76.195, "p95_ms": 72.813}, "yolo11_640_fire_11class": {"avg_ms": 46.472, "count": 127, "max_ms": 62.87, "p95_ms": 57.868}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.945, "count": 127, "max_ms": 79.537, "p95_ms": 73.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.041, "count": 128, "max_ms": 267.665, "p95_ms": 256.449}, "read_interval": {"avg_ms": 236.037, "count": 127, "max_ms": 277.64, "p95_ms": 266.239}, "triton_wait": {"avg_ms": 61.083, "count": 127, "max_ms": 79.549, "p95_ms": 74.193}, "visualize": {"avg_ms": 10.846, "count": 127, "max_ms": 18.37, "p95_ms": 12.222}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 333}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6096.516, "limit_mb": 8796093022207.996}, "detail": {"pid": 13440, "restart_count": 76, "start_count": 77}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:29:14+09:00", "last_error_summary": "[h264 @ 0x55899bd9ddc0] error while decoding MB 12 37, bytestream -36", "last_exit_at": "2026-07-01T14:29:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:29:16+09:00", "pid": 13440, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.082, "name": "ffmpeg", "pid": 13440, "rss_mb": 139.082, "state": "S (sleeping)", "threads": 106}, "restart_count": 76, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.924, "sample_count": 1342, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40034, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:29:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.798, "last_frame_pushed_at": "2026-07-01T14:29:14+09:00", "last_frame_read_age_sec": 5.01, "last_frame_read_at": "2026-07-01T14:29:13+09:00", "mjpeg": {"age_sec": 4.783, "exists": true, "mtime": "2026-07-01T14:29:14.201946+09:00", "mtime_epoch": 1782883754.2019455, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198257}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:29:18+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.727, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.65, "pushed": 3.65, "read": 3.616}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.449, "count": 108, "max_ms": 13.765, "p95_ms": 12.035}, "gstreamer_push": {"avg_ms": 0.861, "count": 109, "max_ms": 0.976, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.298, "count": 108, "max_ms": 43.838, "p95_ms": 41.652}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.844, "count": 108, "max_ms": 76.195, "p95_ms": 71.468}, "yolo11_640_fire_11class": {"avg_ms": 45.77, "count": 108, "max_ms": 61.403, "p95_ms": 56.949}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.513, "count": 108, "max_ms": 79.537, "p95_ms": 72.57}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.378, "count": 109, "max_ms": 267.665, "p95_ms": 254.193}, "read_interval": {"avg_ms": 232.353, "count": 108, "max_ms": 277.64, "p95_ms": 263.531}, "triton_wait": {"avg_ms": 60.646, "count": 108, "max_ms": 79.549, "p95_ms": 73.966}, "visualize": {"avg_ms": 10.697, "count": 108, "max_ms": 18.37, "p95_ms": 12.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.864, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 338}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5989.398, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:29:14+09:00", "last_error_summary": "[h264 @ 0x55899bd9ddc0] error while decoding MB 12 37, bytestream -36", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:29:14+09:00", "last_error_summary": "[h264 @ 0x55899bd9ddc0] error while decoding MB 12 37, bytestream -36", "last_exit_at": "2026-07-01T14:31:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.453, "last_started_at": "2026-07-01T14:29:16+09:00", "pid": 13440, "process": {"exists": false, "pid": 13440}, "restart_count": 76, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.041, "sample_count": 1354, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40629, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:31:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T14:31:25+09:00", "last_frame_read_age_sec": 0.224, "last_frame_read_at": "2026-07-01T14:31:25+09:00", "mjpeg": {"age_sec": 0.215, "exists": true, "mtime": "2026-07-01T14:31:25.221038+09:00", "mtime_epoch": 1782883885.221038, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201798}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:31:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.571, "pushed": 4.571, "read": 4.538}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.73, "count": 136, "max_ms": 12.072, "p95_ms": 10.714}, "gstreamer_push": {"avg_ms": 0.858, "count": 137, "max_ms": 1.344, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.746, "count": 136, "max_ms": 47.335, "p95_ms": 38.65}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.278, "count": 136, "max_ms": 76.015, "p95_ms": 72.426}, "yolo11_640_fire_11class": {"avg_ms": 46.367, "count": 136, "max_ms": 63.254, "p95_ms": 57.896}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.651, "count": 136, "max_ms": 81.019, "p95_ms": 73.606}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.597, "count": 137, "max_ms": 238.041, "p95_ms": 234.278}, "read_interval": {"avg_ms": 220.345, "count": 136, "max_ms": 247.585, "p95_ms": 243.649}, "triton_wait": {"avg_ms": 60.865, "count": 136, "max_ms": 81.032, "p95_ms": 74.923}, "visualize": {"avg_ms": 10.027, "count": 136, "max_ms": 11.636, "p95_ms": 10.875}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 338}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5989.391, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:29:14+09:00", "last_error_summary": "[h264 @ 0x55899bd9ddc0] error while decoding MB 12 37, bytestream -36", "last_exit_at": "2026-07-01T14:31:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.456, "last_started_at": "2026-07-01T14:29:16+09:00", "pid": 13440, "process": {"exists": false, "pid": 13440}, "restart_count": 76, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.041, "sample_count": 1354, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40629, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:31:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T14:31:25+09:00", "last_frame_read_age_sec": 0.228, "last_frame_read_at": "2026-07-01T14:31:25+09:00", "mjpeg": {"age_sec": 0.218, "exists": true, "mtime": "2026-07-01T14:31:25.221038+09:00", "mtime_epoch": 1782883885.221038, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201798}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:31:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.571, "pushed": 4.571, "read": 4.537}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.73, "count": 136, "max_ms": 12.072, "p95_ms": 10.714}, "gstreamer_push": {"avg_ms": 0.858, "count": 137, "max_ms": 1.344, "p95_ms": 0.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.746, "count": 136, "max_ms": 47.335, "p95_ms": 38.65}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.278, "count": 136, "max_ms": 76.015, "p95_ms": 72.426}, "yolo11_640_fire_11class": {"avg_ms": 46.367, "count": 136, "max_ms": 63.254, "p95_ms": 57.896}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.651, "count": 136, "max_ms": 81.019, "p95_ms": 73.606}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.597, "count": 137, "max_ms": 238.041, "p95_ms": 234.278}, "read_interval": {"avg_ms": 220.345, "count": 136, "max_ms": 247.585, "p95_ms": 243.649}, "triton_wait": {"avg_ms": 60.865, "count": 136, "max_ms": 81.032, "p95_ms": 74.923}, "visualize": {"avg_ms": 10.027, "count": 136, "max_ms": 11.636, "p95_ms": 10.875}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 338}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6096.445, "limit_mb": 8796093022207.996}, "detail": {"pid": 13547, "restart_count": 77, "start_count": 78}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:29:14+09:00", "last_error_summary": "[h264 @ 0x55899bd9ddc0] error while decoding MB 12 37, bytestream -36", "last_exit_at": "2026-07-01T14:31:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:31:27+09:00", "pid": 13547, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.785, "name": "ffmpeg", "pid": 13547, "rss_mb": 136.785, "state": "S (sleeping)", "threads": 106}, "restart_count": 77, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.278, "sample_count": 1333, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40629, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:31:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.525, "last_frame_pushed_at": "2026-07-01T14:31:25+09:00", "last_frame_read_age_sec": 4.744, "last_frame_read_at": "2026-07-01T14:31:25+09:00", "mjpeg": {"age_sec": 4.507, "exists": true, "mtime": "2026-07-01T14:31:25.449043+09:00", "mtime_epoch": 1782883885.4490433, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201932}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:31:29+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2369.984, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.891, "pushed": 3.891, "read": 3.858}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.754, "count": 116, "max_ms": 12.072, "p95_ms": 10.714}, "gstreamer_push": {"avg_ms": 0.859, "count": 116, "max_ms": 1.344, "p95_ms": 0.888}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.641, "count": 116, "max_ms": 46.049, "p95_ms": 38.65}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.913, "count": 116, "max_ms": 76.015, "p95_ms": 72.426}, "yolo11_640_fire_11class": {"avg_ms": 46.088, "count": 116, "max_ms": 61.586, "p95_ms": 57.302}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.128, "count": 116, "max_ms": 80.561, "p95_ms": 73.606}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.262, "count": 116, "max_ms": 238.041, "p95_ms": 232.651}, "read_interval": {"avg_ms": 219.995, "count": 115, "max_ms": 247.585, "p95_ms": 243.382}, "triton_wait": {"avg_ms": 60.351, "count": 116, "max_ms": 80.573, "p95_ms": 74.923}, "visualize": {"avg_ms": 10.041, "count": 116, "max_ms": 11.636, "p95_ms": 10.93}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.809, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 343}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5991.227, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a260adcf80] error while decoding MB 24 16, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:33:33+09:00", "last_error_summary": "[h264 @ 0x55a260adcf80] error while decoding MB 24 16, bytestream -5", "last_exit_at": "2026-07-01T14:31:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.696, "last_started_at": "2026-07-01T14:31:27+09:00", "pid": 13547, "process": {"exists": false, "pid": 13547}, "restart_count": 77, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.337, "sample_count": 1361, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41220, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:33:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.051, "last_frame_pushed_at": "2026-07-01T14:33:33+09:00", "last_frame_read_age_sec": 0.044, "last_frame_read_at": "2026-07-01T14:33:33+09:00", "mjpeg": {"age_sec": 0.032, "exists": true, "mtime": "2026-07-01T14:33:33.619952+09:00", "mtime_epoch": 1782884013.6199522, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195463}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:33:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2370.242, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.904, "pushed": 4.904, "read": 4.904}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.887, "count": 146, "max_ms": 11.505, "p95_ms": 9.661}, "gstreamer_push": {"avg_ms": 0.856, "count": 147, "max_ms": 0.995, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.655, "count": 146, "max_ms": 53.067, "p95_ms": 41.939}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.678, "count": 146, "max_ms": 80.017, "p95_ms": 73.19}, "yolo11_640_fire_11class": {"avg_ms": 47.178, "count": 146, "max_ms": 62.84, "p95_ms": 59.3}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.848, "count": 146, "max_ms": 81.877, "p95_ms": 73.649}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.647, "count": 147, "max_ms": 229.854, "p95_ms": 218.823}, "read_interval": {"avg_ms": 204.995, "count": 147, "max_ms": 239.782, "p95_ms": 228.518}, "triton_wait": {"avg_ms": 61.194, "count": 146, "max_ms": 81.89, "p95_ms": 74.199}, "visualize": {"avg_ms": 9.315, "count": 146, "max_ms": 12.464, "p95_ms": 10.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 343}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5990.715, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:33:33+09:00", "last_error_summary": "[h264 @ 0x55a260adcf80] error while decoding MB 24 16, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:33:33+09:00", "last_error_summary": "[h264 @ 0x55a260adcf80] error while decoding MB 24 16, bytestream -5", "last_exit_at": "2026-07-01T14:33:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.86, "last_started_at": "2026-07-01T14:31:27+09:00", "pid": 13547, "process": {"exists": false, "pid": 13547}, "restart_count": 77, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.337, "sample_count": 1361, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41221, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:33:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:33:33+09:00", "last_frame_read_age_sec": 0.208, "last_frame_read_at": "2026-07-01T14:33:33+09:00", "mjpeg": {"age_sec": 0.196, "exists": true, "mtime": "2026-07-01T14:33:33.619952+09:00", "mtime_epoch": 1782884013.6199522, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195463}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:33:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2370.242, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.91, "pushed": 4.91, "read": 4.877}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.904, "count": 146, "max_ms": 11.505, "p95_ms": 9.702}, "gstreamer_push": {"avg_ms": 0.856, "count": 147, "max_ms": 0.995, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.679, "count": 146, "max_ms": 53.067, "p95_ms": 41.939}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.702, "count": 146, "max_ms": 80.017, "p95_ms": 73.19}, "yolo11_640_fire_11class": {"avg_ms": 47.218, "count": 146, "max_ms": 62.84, "p95_ms": 59.3}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.878, "count": 146, "max_ms": 81.877, "p95_ms": 73.649}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.672, "count": 147, "max_ms": 229.854, "p95_ms": 218.823}, "read_interval": {"avg_ms": 204.971, "count": 146, "max_ms": 239.782, "p95_ms": 228.518}, "triton_wait": {"avg_ms": 61.225, "count": 146, "max_ms": 81.89, "p95_ms": 74.199}, "visualize": {"avg_ms": 9.313, "count": 146, "max_ms": 12.464, "p95_ms": 10.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 343}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5990.707, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:33:33+09:00", "last_error_summary": "[h264 @ 0x55a260adcf80] error while decoding MB 24 16, bytestream -5", "last_exit_at": "2026-07-01T14:33:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.864, "last_started_at": "2026-07-01T14:31:27+09:00", "pid": 13547, "process": {"exists": false, "pid": 13547}, "restart_count": 77, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.337, "sample_count": 1361, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41221, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:33:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:33:33+09:00", "last_frame_read_age_sec": 0.212, "last_frame_read_at": "2026-07-01T14:33:33+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T14:33:33.619952+09:00", "mtime_epoch": 1782884013.6199522, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195463}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:33:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2370.242, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.91, "pushed": 4.91, "read": 4.876}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.904, "count": 146, "max_ms": 11.505, "p95_ms": 9.702}, "gstreamer_push": {"avg_ms": 0.856, "count": 147, "max_ms": 0.995, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.679, "count": 146, "max_ms": 53.067, "p95_ms": 41.939}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.702, "count": 146, "max_ms": 80.017, "p95_ms": 73.19}, "yolo11_640_fire_11class": {"avg_ms": 47.218, "count": 146, "max_ms": 62.84, "p95_ms": 59.3}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.878, "count": 146, "max_ms": 81.877, "p95_ms": 73.649}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.672, "count": 147, "max_ms": 229.854, "p95_ms": 218.823}, "read_interval": {"avg_ms": 204.971, "count": 146, "max_ms": 239.782, "p95_ms": 228.518}, "triton_wait": {"avg_ms": 61.225, "count": 146, "max_ms": 81.89, "p95_ms": 74.199}, "visualize": {"avg_ms": 9.313, "count": 146, "max_ms": 12.464, "p95_ms": 10.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 343}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6102.465, "limit_mb": 8796093022207.996}, "detail": {"pid": 13654, "restart_count": 78, "start_count": 79}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:33:33+09:00", "last_error_summary": "[h264 @ 0x55a260adcf80] error while decoding MB 24 16, bytestream -5", "last_exit_at": "2026-07-01T14:33:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:33:36+09:00", "pid": 13654, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.941, "name": "ffmpeg", "pid": 13654, "rss_mb": 141.941, "state": "S (sleeping)", "threads": 106}, "restart_count": 78, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.337, "sample_count": 1338, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41221, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:33:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.671, "last_frame_pushed_at": "2026-07-01T14:33:33+09:00", "last_frame_read_age_sec": 4.876, "last_frame_read_at": "2026-07-01T14:33:33+09:00", "mjpeg": {"age_sec": 4.655, "exists": true, "mtime": "2026-07-01T14:33:33.827957+09:00", "mtime_epoch": 1782884013.8279567, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187644}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:33:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2370.242, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.153, "pushed": 4.153, "read": 4.119}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.924, "count": 124, "max_ms": 11.505, "p95_ms": 9.763}, "gstreamer_push": {"avg_ms": 0.857, "count": 124, "max_ms": 0.995, "p95_ms": 0.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.913, "count": 124, "max_ms": 53.067, "p95_ms": 42.751}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.829, "count": 124, "max_ms": 80.017, "p95_ms": 73.19}, "yolo11_640_fire_11class": {"avg_ms": 47.202, "count": 124, "max_ms": 62.84, "p95_ms": 58.87}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.077, "count": 124, "max_ms": 81.877, "p95_ms": 74.186}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.408, "count": 124, "max_ms": 229.854, "p95_ms": 221.228}, "read_interval": {"avg_ms": 204.705, "count": 123, "max_ms": 239.782, "p95_ms": 230.72}, "triton_wait": {"avg_ms": 61.418, "count": 124, "max_ms": 81.89, "p95_ms": 74.206}, "visualize": {"avg_ms": 9.293, "count": 124, "max_ms": 12.464, "p95_ms": 10.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.86, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 347}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5991.797, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:33:33+09:00", "last_error_summary": "[h264 @ 0x55a260adcf80] error while decoding MB 24 16, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:33:33+09:00", "last_error_summary": "[h264 @ 0x55a260adcf80] error while decoding MB 24 16, bytestream -5", "last_exit_at": "2026-07-01T14:35:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 117.361, "last_started_at": "2026-07-01T14:33:36+09:00", "pid": 13654, "process": {"exists": false, "pid": 13654}, "restart_count": 78, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.184, "sample_count": 1361, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41751, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:35:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:35:33+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T14:35:33+09:00", "mjpeg": {"age_sec": 0.2, "exists": true, "mtime": "2026-07-01T14:35:33.642594+09:00", "mtime_epoch": 1782884133.6425943, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:35:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2370.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.473, "pushed": 4.473, "read": 4.44}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.834, "count": 133, "max_ms": 12.586, "p95_ms": 10.514}, "gstreamer_push": {"avg_ms": 0.853, "count": 134, "max_ms": 0.901, "p95_ms": 0.884}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.888, "count": 133, "max_ms": 46.194, "p95_ms": 35.676}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.949, "count": 133, "max_ms": 79.301, "p95_ms": 76.089}, "yolo11_640_fire_11class": {"avg_ms": 46.613, "count": 133, "max_ms": 66.076, "p95_ms": 58.64}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.54, "count": 133, "max_ms": 78.631, "p95_ms": 74.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 215.314, "count": 134, "max_ms": 243.918, "p95_ms": 235.722}, "read_interval": {"avg_ms": 225.248, "count": 133, "max_ms": 255.144, "p95_ms": 245.494}, "triton_wait": {"avg_ms": 61.871, "count": 133, "max_ms": 79.314, "p95_ms": 76.859}, "visualize": {"avg_ms": 10.275, "count": 133, "max_ms": 11.959, "p95_ms": 11.106}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 347}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5991.789, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:33:33+09:00", "last_error_summary": "[h264 @ 0x55a260adcf80] error while decoding MB 24 16, bytestream -5", "last_exit_at": "2026-07-01T14:35:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 117.364, "last_started_at": "2026-07-01T14:33:36+09:00", "pid": 13654, "process": {"exists": false, "pid": 13654}, "restart_count": 78, "restart_count_per_hour": 27, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.184, "sample_count": 1361, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41751, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:35:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:35:33+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T14:35:33+09:00", "mjpeg": {"age_sec": 0.204, "exists": true, "mtime": "2026-07-01T14:35:33.642594+09:00", "mtime_epoch": 1782884133.6425943, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:35:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2370.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.472, "pushed": 4.472, "read": 4.439}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.834, "count": 133, "max_ms": 12.586, "p95_ms": 10.514}, "gstreamer_push": {"avg_ms": 0.853, "count": 134, "max_ms": 0.901, "p95_ms": 0.884}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.888, "count": 133, "max_ms": 46.194, "p95_ms": 35.676}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.949, "count": 133, "max_ms": 79.301, "p95_ms": 76.089}, "yolo11_640_fire_11class": {"avg_ms": 46.613, "count": 133, "max_ms": 66.076, "p95_ms": 58.64}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.54, "count": 133, "max_ms": 78.631, "p95_ms": 74.63}}, "truncated_count": 0}, "processing_total": {"avg_ms": 215.314, "count": 134, "max_ms": 243.918, "p95_ms": 235.722}, "read_interval": {"avg_ms": 225.248, "count": 133, "max_ms": 255.144, "p95_ms": 245.494}, "triton_wait": {"avg_ms": 61.871, "count": 133, "max_ms": 79.314, "p95_ms": 76.859}, "visualize": {"avg_ms": 10.275, "count": 133, "max_ms": 11.959, "p95_ms": 11.106}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 347}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6097.812, "limit_mb": 8796093022207.996}, "detail": {"pid": 13761, "restart_count": 79, "start_count": 80}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:33:33+09:00", "last_error_summary": "[h264 @ 0x55a260adcf80] error while decoding MB 24 16, bytestream -5", "last_exit_at": "2026-07-01T14:35:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:35:36+09:00", "pid": 13761, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.145, "name": "ffmpeg", "pid": 13761, "rss_mb": 137.145, "state": "S (sleeping)", "threads": 106}, "restart_count": 79, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.248, "sample_count": 1338, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41751, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:35:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 5.0, "last_frame_pushed_at": "2026-07-01T14:35:33+09:00", "last_frame_read_age_sec": 5.207, "last_frame_read_at": "2026-07-01T14:35:33+09:00", "mjpeg": {"age_sec": 4.981, "exists": true, "mtime": "2026-07-01T14:35:33.858599+09:00", "mtime_epoch": 1782884133.8585992, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195558}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:35:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2370.758, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.75, "pushed": 3.75, "read": 3.716}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.711, "count": 111, "max_ms": 10.796, "p95_ms": 10.281}, "gstreamer_push": {"avg_ms": 0.853, "count": 112, "max_ms": 0.901, "p95_ms": 0.879}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 31.666, "count": 111, "max_ms": 46.194, "p95_ms": 35.606}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.899, "count": 111, "max_ms": 79.301, "p95_ms": 76.748}, "yolo11_640_fire_11class": {"avg_ms": 46.227, "count": 111, "max_ms": 66.076, "p95_ms": 57.028}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.483, "count": 111, "max_ms": 78.631, "p95_ms": 74.774}}, "truncated_count": 0}, "processing_total": {"avg_ms": 214.199, "count": 112, "max_ms": 243.918, "p95_ms": 235.074}, "read_interval": {"avg_ms": 224.146, "count": 111, "max_ms": 255.144, "p95_ms": 244.614}, "triton_wait": {"avg_ms": 61.753, "count": 111, "max_ms": 79.314, "p95_ms": 77.096}, "visualize": {"avg_ms": 10.205, "count": 111, "max_ms": 11.959, "p95_ms": 11.04}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 351}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5992.703, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b7551d55c0] cabac decode of qscale diff failed at 55 29 [h264 @ 0x55b7551d55c0] error while decoding MB 55 29, bytestream 7354", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:37:27+09:00", "last_error_summary": "[h264 @ 0x55b7551d55c0] cabac decode of qscale diff failed at 55 29 [h264 @ 0x55b7551d55c0] error while decoding MB 55 29, bytestream 7354", "last_exit_at": "2026-07-01T14:35:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.331, "last_started_at": "2026-07-01T14:35:36+09:00", "pid": 13761, "process": {"exists": false, "pid": 13761}, "restart_count": 79, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.854, "sample_count": 1317, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42211, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:37:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-01T14:37:27+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-01T14:37:27+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-01T14:37:27.113035+09:00", "mtime_epoch": 1782884247.1130352, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202861}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:37:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2370.758, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.536, "pushed": 4.536, "read": 4.536}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.634, "count": 135, "max_ms": 12.311, "p95_ms": 10.531}, "gstreamer_push": {"avg_ms": 0.869, "count": 136, "max_ms": 1.087, "p95_ms": 0.912}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.279, "count": 135, "max_ms": 47.38, "p95_ms": 43.803}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.4, "count": 135, "max_ms": 76.938, "p95_ms": 73.106}, "yolo11_640_fire_11class": {"avg_ms": 48.238, "count": 135, "max_ms": 61.197, "p95_ms": 60.174}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.043, "count": 135, "max_ms": 80.507, "p95_ms": 77.373}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.962, "count": 136, "max_ms": 252.043, "p95_ms": 238.656}, "read_interval": {"avg_ms": 221.818, "count": 136, "max_ms": 261.487, "p95_ms": 249.179}, "triton_wait": {"avg_ms": 62.182, "count": 135, "max_ms": 80.52, "p95_ms": 77.385}, "visualize": {"avg_ms": 10.021, "count": 135, "max_ms": 11.525, "p95_ms": 11.094}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 351}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5991.883, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:37:27+09:00", "last_error_summary": "[h264 @ 0x55b7551d55c0] cabac decode of qscale diff failed at 55 29 [h264 @ 0x55b7551d55c0] error while decoding MB 55 29, bytestream 7354", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:37:27+09:00", "last_error_summary": "[h264 @ 0x55b7551d55c0] cabac decode of qscale diff failed at 55 29 [h264 @ 0x55b7551d55c0] error while decoding MB 55 29, bytestream 7354", "last_exit_at": "2026-07-01T14:37:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.507, "last_started_at": "2026-07-01T14:35:36+09:00", "pid": 13761, "process": {"exists": false, "pid": 13761}, "restart_count": 79, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.98, "sample_count": 1317, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42212, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:37:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:37:27+09:00", "last_frame_read_age_sec": 0.24, "last_frame_read_at": "2026-07-01T14:37:27+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-01T14:37:27.113035+09:00", "mtime_epoch": 1782884247.1130352, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202861}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:37:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2370.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.545, "pushed": 4.545, "read": 4.512}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.634, "count": 135, "max_ms": 12.311, "p95_ms": 10.531}, "gstreamer_push": {"avg_ms": 0.868, "count": 136, "max_ms": 1.087, "p95_ms": 0.912}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.274, "count": 135, "max_ms": 47.38, "p95_ms": 43.803}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.451, "count": 135, "max_ms": 76.938, "p95_ms": 73.106}, "yolo11_640_fire_11class": {"avg_ms": 48.24, "count": 135, "max_ms": 61.197, "p95_ms": 60.174}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.115, "count": 135, "max_ms": 80.507, "p95_ms": 77.373}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.842, "count": 136, "max_ms": 244.878, "p95_ms": 238.162}, "read_interval": {"avg_ms": 221.524, "count": 135, "max_ms": 255.523, "p95_ms": 248.224}, "triton_wait": {"avg_ms": 62.254, "count": 135, "max_ms": 80.52, "p95_ms": 77.385}, "visualize": {"avg_ms": 10.023, "count": 135, "max_ms": 11.525, "p95_ms": 11.094}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 351}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5992.102, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:37:27+09:00", "last_error_summary": "[h264 @ 0x55b7551d55c0] cabac decode of qscale diff failed at 55 29 [h264 @ 0x55b7551d55c0] error while decoding MB 55 29, bytestream 7354", "last_exit_at": "2026-07-01T14:37:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.51, "last_started_at": "2026-07-01T14:35:36+09:00", "pid": 13761, "process": {"exists": false, "pid": 13761}, "restart_count": 79, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.98, "sample_count": 1317, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42212, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:37:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:37:27+09:00", "last_frame_read_age_sec": 0.244, "last_frame_read_at": "2026-07-01T14:37:27+09:00", "mjpeg": {"age_sec": 0.236, "exists": true, "mtime": "2026-07-01T14:37:27.113035+09:00", "mtime_epoch": 1782884247.1130352, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202861}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:37:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2370.758, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.545, "pushed": 4.545, "read": 4.511}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.634, "count": 135, "max_ms": 12.311, "p95_ms": 10.531}, "gstreamer_push": {"avg_ms": 0.868, "count": 136, "max_ms": 1.087, "p95_ms": 0.912}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.274, "count": 135, "max_ms": 47.38, "p95_ms": 43.803}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.451, "count": 135, "max_ms": 76.938, "p95_ms": 73.106}, "yolo11_640_fire_11class": {"avg_ms": 48.24, "count": 135, "max_ms": 61.197, "p95_ms": 60.174}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.115, "count": 135, "max_ms": 80.507, "p95_ms": 77.373}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.842, "count": 136, "max_ms": 244.878, "p95_ms": 238.162}, "read_interval": {"avg_ms": 221.524, "count": 135, "max_ms": 255.523, "p95_ms": 248.224}, "triton_wait": {"avg_ms": 62.254, "count": 135, "max_ms": 80.52, "p95_ms": 77.385}, "visualize": {"avg_ms": 10.023, "count": 135, "max_ms": 11.525, "p95_ms": 11.094}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 351}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6105.617, "limit_mb": 8796093022207.996}, "detail": {"pid": 13868, "restart_count": 80, "start_count": 81}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:37:27+09:00", "last_error_summary": "[h264 @ 0x55b7551d55c0] cabac decode of qscale diff failed at 55 29 [h264 @ 0x55b7551d55c0] error while decoding MB 55 29, bytestream 7354", "last_exit_at": "2026-07-01T14:37:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:37:29+09:00", "pid": 13868, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.359, "name": "ffmpeg", "pid": 13868, "rss_mb": 144.359, "state": "S (sleeping)", "threads": 106}, "restart_count": 80, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 230.565, "sample_count": 1297, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42212, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:37:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.234, "last_frame_pushed_at": "2026-07-01T14:37:27+09:00", "last_frame_read_age_sec": 4.471, "last_frame_read_at": "2026-07-01T14:37:27+09:00", "mjpeg": {"age_sec": 4.219, "exists": true, "mtime": "2026-07-01T14:37:27.357040+09:00", "mtime_epoch": 1782884247.3570404, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199230}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:37:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2370.758, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.915, "pushed": 3.915, "read": 3.882}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.53, "count": 117, "max_ms": 12.311, "p95_ms": 10.384}, "gstreamer_push": {"avg_ms": 0.868, "count": 117, "max_ms": 1.087, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.473, "count": 117, "max_ms": 47.38, "p95_ms": 44.48}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.049, "count": 117, "max_ms": 76.938, "p95_ms": 73.428}, "yolo11_640_fire_11class": {"avg_ms": 48.623, "count": 117, "max_ms": 61.197, "p95_ms": 60.381}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.673, "count": 117, "max_ms": 80.507, "p95_ms": 77.708}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.062, "count": 117, "max_ms": 244.878, "p95_ms": 238.656}, "read_interval": {"avg_ms": 220.697, "count": 116, "max_ms": 255.523, "p95_ms": 249.179}, "triton_wait": {"avg_ms": 62.831, "count": 117, "max_ms": 80.52, "p95_ms": 77.72}, "visualize": {"avg_ms": 9.947, "count": 117, "max_ms": 11.525, "p95_ms": 11.023}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 356}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5993.855, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5572603aa140] error while decoding MB 36 20, bytestream -30", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:39:36+09:00", "last_error_summary": "[h264 @ 0x5572603aa140] error while decoding MB 36 20, bytestream -30", "last_exit_at": "2026-07-01T14:37:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.195, "last_started_at": "2026-07-01T14:37:29+09:00", "pid": 13868, "process": {"exists": false, "pid": 13868}, "restart_count": 80, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 232.62, "sample_count": 1309, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42801, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:39:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.048, "last_frame_pushed_at": "2026-07-01T14:39:36+09:00", "last_frame_read_age_sec": 0.04, "last_frame_read_at": "2026-07-01T14:39:36+09:00", "mjpeg": {"age_sec": 0.027, "exists": true, "mtime": "2026-07-01T14:39:36.743770+09:00", "mtime_epoch": 1782884376.74377, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207861}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:39:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.016, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.737, "pushed": 4.737, "read": 4.737}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.328, "count": 141, "max_ms": 11.628, "p95_ms": 10.417}, "gstreamer_push": {"avg_ms": 0.871, "count": 142, "max_ms": 1.272, "p95_ms": 0.91}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.256, "count": 141, "max_ms": 46.659, "p95_ms": 40.499}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.671, "count": 141, "max_ms": 76.606, "p95_ms": 72.506}, "yolo11_640_fire_11class": {"avg_ms": 47.084, "count": 141, "max_ms": 60.483, "p95_ms": 56.27}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.501, "count": 141, "max_ms": 79.813, "p95_ms": 72.75}}, "truncated_count": 0}, "processing_total": {"avg_ms": 202.421, "count": 142, "max_ms": 239.639, "p95_ms": 217.77}, "read_interval": {"avg_ms": 212.282, "count": 142, "max_ms": 250.973, "p95_ms": 228.066}, "triton_wait": {"avg_ms": 61.755, "count": 141, "max_ms": 79.826, "p95_ms": 73.48}, "visualize": {"avg_ms": 9.653, "count": 141, "max_ms": 12.439, "p95_ms": 10.79}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 356}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5992.969, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:39:36+09:00", "last_error_summary": "[h264 @ 0x5572603aa140] error while decoding MB 36 20, bytestream -30", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:39:36+09:00", "last_error_summary": "[h264 @ 0x5572603aa140] error while decoding MB 36 20, bytestream -30", "last_exit_at": "2026-07-01T14:39:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.357, "last_started_at": "2026-07-01T14:37:29+09:00", "pid": 13868, "process": {"exists": false, "pid": 13868}, "restart_count": 80, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 232.62, "sample_count": 1309, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:39:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:39:36+09:00", "last_frame_read_age_sec": 0.202, "last_frame_read_at": "2026-07-01T14:39:36+09:00", "mjpeg": {"age_sec": 0.189, "exists": true, "mtime": "2026-07-01T14:39:36.743770+09:00", "mtime_epoch": 1782884376.74377, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207861}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:39:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.016, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.744, "pushed": 4.744, "read": 4.711}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.321, "count": 141, "max_ms": 11.628, "p95_ms": 10.417}, "gstreamer_push": {"avg_ms": 0.871, "count": 142, "max_ms": 1.272, "p95_ms": 0.91}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.234, "count": 141, "max_ms": 46.659, "p95_ms": 40.499}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.671, "count": 141, "max_ms": 76.606, "p95_ms": 72.506}, "yolo11_640_fire_11class": {"avg_ms": 47.121, "count": 141, "max_ms": 60.483, "p95_ms": 56.27}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.505, "count": 141, "max_ms": 79.813, "p95_ms": 72.75}}, "truncated_count": 0}, "processing_total": {"avg_ms": 202.371, "count": 142, "max_ms": 239.639, "p95_ms": 217.77}, "read_interval": {"avg_ms": 212.265, "count": 141, "max_ms": 250.973, "p95_ms": 228.066}, "triton_wait": {"avg_ms": 61.759, "count": 141, "max_ms": 79.826, "p95_ms": 73.48}, "visualize": {"avg_ms": 9.653, "count": 141, "max_ms": 12.439, "p95_ms": 10.79}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 356}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5992.961, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:39:36+09:00", "last_error_summary": "[h264 @ 0x5572603aa140] error while decoding MB 36 20, bytestream -30", "last_exit_at": "2026-07-01T14:39:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.361, "last_started_at": "2026-07-01T14:37:29+09:00", "pid": 13868, "process": {"exists": false, "pid": 13868}, "restart_count": 80, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 232.62, "sample_count": 1309, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:39:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T14:39:36+09:00", "last_frame_read_age_sec": 0.206, "last_frame_read_at": "2026-07-01T14:39:36+09:00", "mjpeg": {"age_sec": 0.192, "exists": true, "mtime": "2026-07-01T14:39:36.743770+09:00", "mtime_epoch": 1782884376.74377, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207861}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:39:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.016, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.743, "pushed": 4.743, "read": 4.71}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.321, "count": 141, "max_ms": 11.628, "p95_ms": 10.417}, "gstreamer_push": {"avg_ms": 0.871, "count": 142, "max_ms": 1.272, "p95_ms": 0.91}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.234, "count": 141, "max_ms": 46.659, "p95_ms": 40.499}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.671, "count": 141, "max_ms": 76.606, "p95_ms": 72.506}, "yolo11_640_fire_11class": {"avg_ms": 47.121, "count": 141, "max_ms": 60.483, "p95_ms": 56.27}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.505, "count": 141, "max_ms": 79.813, "p95_ms": 72.75}}, "truncated_count": 0}, "processing_total": {"avg_ms": 202.371, "count": 142, "max_ms": 239.639, "p95_ms": 217.77}, "read_interval": {"avg_ms": 212.265, "count": 141, "max_ms": 250.973, "p95_ms": 228.066}, "triton_wait": {"avg_ms": 61.759, "count": 141, "max_ms": 79.826, "p95_ms": 73.48}, "visualize": {"avg_ms": 9.653, "count": 141, "max_ms": 12.439, "p95_ms": 10.79}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 356}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6109.703, "limit_mb": 8796093022207.996}, "detail": {"pid": 13975, "restart_count": 81, "start_count": 82}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:39:36+09:00", "last_error_summary": "[h264 @ 0x5572603aa140] error while decoding MB 36 20, bytestream -30", "last_exit_at": "2026-07-01T14:39:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:39:39+09:00", "pid": 13975, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.824, "name": "ffmpeg", "pid": 13975, "rss_mb": 147.824, "state": "S (sleeping)", "threads": 106}, "restart_count": 81, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.857, "sample_count": 1288, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:39:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.667, "last_frame_pushed_at": "2026-07-01T14:39:36+09:00", "last_frame_read_age_sec": 4.866, "last_frame_read_at": "2026-07-01T14:39:36+09:00", "mjpeg": {"age_sec": 4.649, "exists": true, "mtime": "2026-07-01T14:39:36.947774+09:00", "mtime_epoch": 1782884376.9477742, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199283}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:39:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.016, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.973, "pushed": 3.973, "read": 3.939}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.371, "count": 118, "max_ms": 11.372, "p95_ms": 10.417}, "gstreamer_push": {"avg_ms": 0.874, "count": 119, "max_ms": 1.272, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.148, "count": 118, "max_ms": 46.283, "p95_ms": 40.941}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.558, "count": 118, "max_ms": 76.606, "p95_ms": 73.054}, "yolo11_640_fire_11class": {"avg_ms": 47.146, "count": 118, "max_ms": 60.483, "p95_ms": 57.592}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.626, "count": 118, "max_ms": 79.813, "p95_ms": 73.036}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.453, "count": 119, "max_ms": 239.639, "p95_ms": 218.607}, "read_interval": {"avg_ms": 214.361, "count": 118, "max_ms": 250.973, "p95_ms": 229.4}, "triton_wait": {"avg_ms": 61.882, "count": 118, "max_ms": 79.826, "p95_ms": 74.731}, "visualize": {"avg_ms": 9.755, "count": 118, "max_ms": 12.439, "p95_ms": 10.843}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6206.309, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5618aba9ed00] error while decoding MB 78 2, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:41:47+09:00", "last_error_summary": "[h264 @ 0x5618aba9ed00] error while decoding MB 78 2, bytestream -14", "last_exit_at": "2026-07-01T14:39:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.417, "last_started_at": "2026-07-01T14:39:39+09:00", "pid": 13975, "process": {"exists": false, "pid": 13975}, "restart_count": 81, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.225, "sample_count": 1362, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43390, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:41:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.055, "last_frame_pushed_at": "2026-07-01T14:41:46+09:00", "last_frame_read_age_sec": 0.047, "last_frame_read_at": "2026-07-01T14:41:46+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T14:41:46.974472+09:00", "mtime_epoch": 1782884506.974472, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198948}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:41:47+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.273, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.702, "pushed": 4.702, "read": 4.702}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.323, "count": 140, "max_ms": 14.725, "p95_ms": 11.03}, "gstreamer_push": {"avg_ms": 0.865, "count": 141, "max_ms": 1.224, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.098, "count": 140, "max_ms": 50.767, "p95_ms": 42.021}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.485, "count": 140, "max_ms": 76.473, "p95_ms": 72.996}, "yolo11_640_fire_11class": {"avg_ms": 47.343, "count": 140, "max_ms": 63.749, "p95_ms": 57.38}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.373, "count": 140, "max_ms": 81.743, "p95_ms": 73.318}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.938, "count": 141, "max_ms": 230.145, "p95_ms": 221.779}, "read_interval": {"avg_ms": 213.778, "count": 141, "max_ms": 239.972, "p95_ms": 231.14}, "triton_wait": {"avg_ms": 61.768, "count": 140, "max_ms": 81.755, "p95_ms": 74.362}, "visualize": {"avg_ms": 9.731, "count": 140, "max_ms": 13.402, "p95_ms": 10.677}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6215.398, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:41:47+09:00", "last_error_summary": "[h264 @ 0x5618aba9ed00] error while decoding MB 78 2, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:41:47+09:00", "last_error_summary": "[h264 @ 0x5618aba9ed00] error while decoding MB 78 2, bytestream -14", "last_exit_at": "2026-07-01T14:41:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.569, "last_started_at": "2026-07-01T14:39:39+09:00", "pid": 13975, "process": {"exists": false, "pid": 13975}, "restart_count": 81, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.896, "sample_count": 1362, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:41:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:41:47+09:00", "last_frame_read_age_sec": 0.199, "last_frame_read_at": "2026-07-01T14:41:46+09:00", "mjpeg": {"age_sec": 0.191, "exists": true, "mtime": "2026-07-01T14:41:46.974472+09:00", "mtime_epoch": 1782884506.974472, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198948}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:41:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.712, "pushed": 4.712, "read": 4.678}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.324, "count": 140, "max_ms": 14.725, "p95_ms": 11.03}, "gstreamer_push": {"avg_ms": 0.865, "count": 141, "max_ms": 1.224, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.087, "count": 140, "max_ms": 50.767, "p95_ms": 42.021}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.456, "count": 140, "max_ms": 76.473, "p95_ms": 72.996}, "yolo11_640_fire_11class": {"avg_ms": 47.258, "count": 140, "max_ms": 63.749, "p95_ms": 57.38}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.331, "count": 140, "max_ms": 81.743, "p95_ms": 73.318}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.874, "count": 141, "max_ms": 230.145, "p95_ms": 221.779}, "read_interval": {"avg_ms": 213.784, "count": 140, "max_ms": 239.972, "p95_ms": 231.14}, "triton_wait": {"avg_ms": 61.726, "count": 140, "max_ms": 81.755, "p95_ms": 74.362}, "visualize": {"avg_ms": 9.724, "count": 140, "max_ms": 13.402, "p95_ms": 10.677}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6215.391, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:41:47+09:00", "last_error_summary": "[h264 @ 0x5618aba9ed00] error while decoding MB 78 2, bytestream -14", "last_exit_at": "2026-07-01T14:41:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.573, "last_started_at": "2026-07-01T14:39:39+09:00", "pid": 13975, "process": {"exists": false, "pid": 13975}, "restart_count": 81, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.896, "sample_count": 1362, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:41:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:41:47+09:00", "last_frame_read_age_sec": 0.202, "last_frame_read_at": "2026-07-01T14:41:46+09:00", "mjpeg": {"age_sec": 0.194, "exists": true, "mtime": "2026-07-01T14:41:46.974472+09:00", "mtime_epoch": 1782884506.974472, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198948}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:41:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.711, "pushed": 4.711, "read": 4.678}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.324, "count": 140, "max_ms": 14.725, "p95_ms": 11.03}, "gstreamer_push": {"avg_ms": 0.865, "count": 141, "max_ms": 1.224, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.087, "count": 140, "max_ms": 50.767, "p95_ms": 42.021}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.456, "count": 140, "max_ms": 76.473, "p95_ms": 72.996}, "yolo11_640_fire_11class": {"avg_ms": 47.258, "count": 140, "max_ms": 63.749, "p95_ms": 57.38}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.331, "count": 140, "max_ms": 81.743, "p95_ms": 73.318}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.874, "count": 141, "max_ms": 230.145, "p95_ms": 221.779}, "read_interval": {"avg_ms": 213.784, "count": 140, "max_ms": 239.972, "p95_ms": 231.14}, "triton_wait": {"avg_ms": 61.726, "count": 140, "max_ms": 81.755, "p95_ms": 74.362}, "visualize": {"avg_ms": 9.724, "count": 140, "max_ms": 13.402, "p95_ms": 10.677}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 361}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6271.375, "limit_mb": 8796093022207.996}, "detail": {"pid": 14142, "restart_count": 82, "start_count": 83}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:41:47+09:00", "last_error_summary": "[h264 @ 0x5618aba9ed00] error while decoding MB 78 2, bytestream -14", "last_exit_at": "2026-07-01T14:41:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:41:49+09:00", "pid": 14142, "process": {"exists": true, "fd_count": 4, "hwm_mb": 52.891, "name": "ffmpeg", "pid": 14142, "rss_mb": 52.891, "state": "S (sleeping)", "threads": 1}, "restart_count": 82, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.896, "sample_count": 1342, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:41:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.143, "last_frame_pushed_at": "2026-07-01T14:41:47+09:00", "last_frame_read_age_sec": 4.338, "last_frame_read_at": "2026-07-01T14:41:46+09:00", "mjpeg": {"age_sec": 4.127, "exists": true, "mtime": "2026-07-01T14:41:47.178476+09:00", "mtime_epoch": 1782884507.178476, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:41:51+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.273, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.058, "pushed": 4.058, "read": 4.024}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.359, "count": 121, "max_ms": 14.725, "p95_ms": 11.274}, "gstreamer_push": {"avg_ms": 0.867, "count": 121, "max_ms": 1.224, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.032, "count": 121, "max_ms": 50.767, "p95_ms": 42.021}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.506, "count": 121, "max_ms": 76.473, "p95_ms": 72.996}, "yolo11_640_fire_11class": {"avg_ms": 47.264, "count": 121, "max_ms": 63.749, "p95_ms": 56.543}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.414, "count": 121, "max_ms": 81.743, "p95_ms": 73.788}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.104, "count": 121, "max_ms": 230.145, "p95_ms": 222.141}, "read_interval": {"avg_ms": 214.062, "count": 120, "max_ms": 239.972, "p95_ms": 231.538}, "triton_wait": {"avg_ms": 61.832, "count": 121, "max_ms": 81.755, "p95_ms": 74.492}, "visualize": {"avg_ms": 9.728, "count": 121, "max_ms": 13.402, "p95_ms": 10.725}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.819, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 366}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6066.75, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fd25668ec0] error while decoding MB 23 13, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:43:57+09:00", "last_error_summary": "[h264 @ 0x55fd25668ec0] error while decoding MB 23 13, bytestream -5", "last_exit_at": "2026-07-01T14:41:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.746, "last_started_at": "2026-07-01T14:41:49+09:00", "pid": 14142, "process": {"exists": false, "pid": 14142}, "restart_count": 82, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.149, "sample_count": 1368, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43982, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:43:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-01T14:43:56+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-01T14:43:57+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T14:43:57.009135+09:00", "mtime_epoch": 1782884637.0091348, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203291}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:43:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.531, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.754, "pushed": 4.754, "read": 4.754}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.308, "count": 142, "max_ms": 14.903, "p95_ms": 10.482}, "gstreamer_push": {"avg_ms": 0.872, "count": 142, "max_ms": 1.257, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.81, "count": 142, "max_ms": 49.492, "p95_ms": 40.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.247, "count": 142, "max_ms": 80.952, "p95_ms": 74.093}, "yolo11_640_fire_11class": {"avg_ms": 48.015, "count": 142, "max_ms": 69.066, "p95_ms": 59.347}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.028, "count": 142, "max_ms": 80.103, "p95_ms": 74.235}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.638, "count": 142, "max_ms": 243.636, "p95_ms": 220.726}, "read_interval": {"avg_ms": 211.399, "count": 142, "max_ms": 253.478, "p95_ms": 230.283}, "triton_wait": {"avg_ms": 61.204, "count": 142, "max_ms": 80.964, "p95_ms": 75.243}, "visualize": {"avg_ms": 9.635, "count": 142, "max_ms": 12.957, "p95_ms": 10.503}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 366}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6066.086, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:43:57+09:00", "last_error_summary": "[h264 @ 0x55fd25668ec0] error while decoding MB 23 13, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:43:57+09:00", "last_error_summary": "[h264 @ 0x55fd25668ec0] error while decoding MB 23 13, bytestream -5", "last_exit_at": "2026-07-01T14:43:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.926, "last_started_at": "2026-07-01T14:41:49+09:00", "pid": 14142, "process": {"exists": false, "pid": 14142}, "restart_count": 82, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.149, "sample_count": 1368, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43983, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:43:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T14:43:57+09:00", "last_frame_read_age_sec": 0.228, "last_frame_read_at": "2026-07-01T14:43:57+09:00", "mjpeg": {"age_sec": 0.221, "exists": true, "mtime": "2026-07-01T14:43:57.009135+09:00", "mtime_epoch": 1782884637.0091348, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203291}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:43:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.531, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.762, "pushed": 4.762, "read": 4.728}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.321, "count": 142, "max_ms": 14.903, "p95_ms": 10.57}, "gstreamer_push": {"avg_ms": 0.872, "count": 142, "max_ms": 1.257, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.778, "count": 142, "max_ms": 49.492, "p95_ms": 40.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.274, "count": 142, "max_ms": 80.952, "p95_ms": 74.093}, "yolo11_640_fire_11class": {"avg_ms": 48.006, "count": 142, "max_ms": 69.066, "p95_ms": 59.347}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.038, "count": 142, "max_ms": 80.103, "p95_ms": 74.235}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.797, "count": 142, "max_ms": 243.636, "p95_ms": 220.922}, "read_interval": {"avg_ms": 211.413, "count": 141, "max_ms": 253.478, "p95_ms": 230.283}, "triton_wait": {"avg_ms": 61.214, "count": 142, "max_ms": 80.964, "p95_ms": 75.243}, "visualize": {"avg_ms": 9.642, "count": 142, "max_ms": 12.957, "p95_ms": 10.632}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.822, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 366}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6066.078, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:43:57+09:00", "last_error_summary": "[h264 @ 0x55fd25668ec0] error while decoding MB 23 13, bytestream -5", "last_exit_at": "2026-07-01T14:43:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.929, "last_started_at": "2026-07-01T14:41:49+09:00", "pid": 14142, "process": {"exists": false, "pid": 14142}, "restart_count": 82, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.149, "sample_count": 1368, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43983, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:43:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T14:43:57+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T14:43:57+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T14:43:57.009135+09:00", "mtime_epoch": 1782884637.0091348, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203291}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:43:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.531, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.761, "pushed": 4.761, "read": 4.727}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.321, "count": 142, "max_ms": 14.903, "p95_ms": 10.57}, "gstreamer_push": {"avg_ms": 0.872, "count": 142, "max_ms": 1.257, "p95_ms": 0.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.778, "count": 142, "max_ms": 49.492, "p95_ms": 40.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.274, "count": 142, "max_ms": 80.952, "p95_ms": 74.093}, "yolo11_640_fire_11class": {"avg_ms": 48.006, "count": 142, "max_ms": 69.066, "p95_ms": 59.347}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.038, "count": 142, "max_ms": 80.103, "p95_ms": 74.235}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.797, "count": 142, "max_ms": 243.636, "p95_ms": 220.922}, "read_interval": {"avg_ms": 211.413, "count": 141, "max_ms": 253.478, "p95_ms": 230.283}, "triton_wait": {"avg_ms": 61.214, "count": 142, "max_ms": 80.964, "p95_ms": 75.243}, "visualize": {"avg_ms": 9.642, "count": 142, "max_ms": 12.957, "p95_ms": 10.632}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.826, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 366}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6174.062, "limit_mb": 8796093022207.996}, "detail": {"pid": 14249, "restart_count": 83, "start_count": 84}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:43:57+09:00", "last_error_summary": "[h264 @ 0x55fd25668ec0] error while decoding MB 23 13, bytestream -5", "last_exit_at": "2026-07-01T14:43:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:43:59+09:00", "pid": 14249, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.09, "name": "ffmpeg", "pid": 14249, "rss_mb": 139.09, "state": "S (sleeping)", "threads": 106}, "restart_count": 83, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.149, "sample_count": 1346, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43983, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:43:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.674, "last_frame_pushed_at": "2026-07-01T14:43:57+09:00", "last_frame_read_age_sec": 4.896, "last_frame_read_at": "2026-07-01T14:43:57+09:00", "mjpeg": {"age_sec": 4.657, "exists": true, "mtime": "2026-07-01T14:43:57.241140+09:00", "mtime_epoch": 1782884637.2411397, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197608}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:44:01+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2371.531, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.023, "pushed": 4.023, "read": 3.99}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.304, "count": 120, "max_ms": 14.903, "p95_ms": 10.72}, "gstreamer_push": {"avg_ms": 0.875, "count": 120, "max_ms": 1.257, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 32.905, "count": 120, "max_ms": 49.492, "p95_ms": 40.588}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.37, "count": 120, "max_ms": 80.952, "p95_ms": 74.633}, "yolo11_640_fire_11class": {"avg_ms": 47.928, "count": 120, "max_ms": 69.066, "p95_ms": 57.713}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.137, "count": 120, "max_ms": 80.103, "p95_ms": 74.673}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.739, "count": 120, "max_ms": 243.636, "p95_ms": 223.364}, "read_interval": {"avg_ms": 211.333, "count": 119, "max_ms": 253.478, "p95_ms": 234.693}, "triton_wait": {"avg_ms": 61.336, "count": 120, "max_ms": 80.964, "p95_ms": 77.089}, "visualize": {"avg_ms": 9.624, "count": 120, "max_ms": 12.957, "p95_ms": 10.685}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.826, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6068.434, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c7a2085d00] cabac decode of qscale diff failed at 8 15 [h264 @ 0x55c7a2085d00] error while decoding MB 8 15, bytestream 367", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:46:00+09:00", "last_error_summary": "[h264 @ 0x55c7a2085d00] cabac decode of qscale diff failed at 8 15 [h264 @ 0x55c7a2085d00] error while decoding MB 8 15, bytestream 367", "last_exit_at": "2026-07-01T14:43:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.628, "last_started_at": "2026-07-01T14:43:59+09:00", "pid": 14249, "process": {"exists": false, "pid": 14249}, "restart_count": 83, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.143, "sample_count": 1370, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44543, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:46:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T14:46:00+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T14:46:00+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T14:46:00.483638+09:00", "mtime_epoch": 1782884760.483638, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200610}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:46:00+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2372.047, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.695, "pushed": 4.695, "read": 4.695}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.255, "count": 140, "max_ms": 12.669, "p95_ms": 10.212}, "gstreamer_push": {"avg_ms": 0.854, "count": 140, "max_ms": 1.058, "p95_ms": 0.896}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.029, "count": 140, "max_ms": 46.418, "p95_ms": 42.113}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.205, "count": 140, "max_ms": 78.803, "p95_ms": 73.221}, "yolo11_640_fire_11class": {"avg_ms": 47.163, "count": 140, "max_ms": 61.716, "p95_ms": 57.871}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.727, "count": 140, "max_ms": 81.936, "p95_ms": 74.893}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.2, "count": 140, "max_ms": 245.513, "p95_ms": 232.351}, "read_interval": {"avg_ms": 213.99, "count": 140, "max_ms": 255.198, "p95_ms": 241.501}, "triton_wait": {"avg_ms": 62.042, "count": 140, "max_ms": 81.948, "p95_ms": 75.684}, "visualize": {"avg_ms": 9.699, "count": 140, "max_ms": 15.06, "p95_ms": 10.953}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.821, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6067.578, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:46:00+09:00", "last_error_summary": "[h264 @ 0x55c7a2085d00] cabac decode of qscale diff failed at 8 15 [h264 @ 0x55c7a2085d00] error while decoding MB 8 15, bytestream 367", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:46:00+09:00", "last_error_summary": "[h264 @ 0x55c7a2085d00] cabac decode of qscale diff failed at 8 15 [h264 @ 0x55c7a2085d00] error while decoding MB 8 15, bytestream 367", "last_exit_at": "2026-07-01T14:46:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.773, "last_started_at": "2026-07-01T14:43:59+09:00", "pid": 14249, "process": {"exists": false, "pid": 14249}, "restart_count": 83, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.143, "sample_count": 1371, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44544, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:46:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:46:00+09:00", "last_frame_read_age_sec": 0.199, "last_frame_read_at": "2026-07-01T14:46:00+09:00", "mjpeg": {"age_sec": 0.186, "exists": true, "mtime": "2026-07-01T14:46:00.483638+09:00", "mtime_epoch": 1782884760.483638, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200610}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:46:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2372.047, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.705, "pushed": 4.705, "read": 4.672}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.253, "count": 140, "max_ms": 12.669, "p95_ms": 10.212}, "gstreamer_push": {"avg_ms": 0.854, "count": 141, "max_ms": 1.058, "p95_ms": 0.896}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.004, "count": 140, "max_ms": 46.418, "p95_ms": 42.113}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.284, "count": 140, "max_ms": 78.803, "p95_ms": 73.221}, "yolo11_640_fire_11class": {"avg_ms": 47.235, "count": 140, "max_ms": 61.716, "p95_ms": 58.384}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.772, "count": 140, "max_ms": 81.936, "p95_ms": 74.893}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.128, "count": 141, "max_ms": 245.513, "p95_ms": 232.351}, "read_interval": {"avg_ms": 213.99, "count": 140, "max_ms": 255.198, "p95_ms": 241.501}, "triton_wait": {"avg_ms": 62.088, "count": 140, "max_ms": 81.948, "p95_ms": 75.684}, "visualize": {"avg_ms": 9.702, "count": 140, "max_ms": 15.06, "p95_ms": 10.953}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6067.691, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:46:00+09:00", "last_error_summary": "[h264 @ 0x55c7a2085d00] cabac decode of qscale diff failed at 8 15 [h264 @ 0x55c7a2085d00] error while decoding MB 8 15, bytestream 367", "last_exit_at": "2026-07-01T14:46:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.776, "last_started_at": "2026-07-01T14:43:59+09:00", "pid": 14249, "process": {"exists": false, "pid": 14249}, "restart_count": 83, "restart_count_per_hour": 28, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.143, "sample_count": 1371, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44544, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:46:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T14:46:00+09:00", "last_frame_read_age_sec": 0.202, "last_frame_read_at": "2026-07-01T14:46:00+09:00", "mjpeg": {"age_sec": 0.19, "exists": true, "mtime": "2026-07-01T14:46:00.483638+09:00", "mtime_epoch": 1782884760.483638, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200610}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:46:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2372.047, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 141, "pushed_frames": 141, "read_frames": 140, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.705, "pushed": 4.705, "read": 4.672}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.253, "count": 140, "max_ms": 12.669, "p95_ms": 10.212}, "gstreamer_push": {"avg_ms": 0.854, "count": 141, "max_ms": 1.058, "p95_ms": 0.896}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.004, "count": 140, "max_ms": 46.418, "p95_ms": 42.113}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.284, "count": 140, "max_ms": 78.803, "p95_ms": 73.221}, "yolo11_640_fire_11class": {"avg_ms": 47.235, "count": 140, "max_ms": 61.716, "p95_ms": 58.384}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.772, "count": 140, "max_ms": 81.936, "p95_ms": 74.893}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.128, "count": 141, "max_ms": 245.513, "p95_ms": 232.351}, "read_interval": {"avg_ms": 213.99, "count": 140, "max_ms": 255.198, "p95_ms": 241.501}, "triton_wait": {"avg_ms": 62.088, "count": 140, "max_ms": 81.948, "p95_ms": 75.684}, "visualize": {"avg_ms": 9.702, "count": 140, "max_ms": 15.06, "p95_ms": 10.953}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_115105_0a9743f8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6174.859, "limit_mb": 8796093022207.996}, "detail": {"pid": 14356, "restart_count": 84, "start_count": 85}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:46:00+09:00", "last_error_summary": "[h264 @ 0x55c7a2085d00] cabac decode of qscale diff failed at 8 15 [h264 @ 0x55c7a2085d00] error while decoding MB 8 15, bytestream 367", "last_exit_at": "2026-07-01T14:46:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:46:03+09:00", "pid": 14356, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.297, "name": "ffmpeg", "pid": 14356, "rss_mb": 139.297, "state": "S (sleeping)", "threads": 106}, "restart_count": 84, "restart_count_per_hour": 29, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.15, "sample_count": 1349, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44544, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:46:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "0a9743f8", "last_frame_pushed_age_sec": 4.684, "last_frame_pushed_at": "2026-07-01T14:46:00+09:00", "last_frame_read_age_sec": 4.88, "last_frame_read_at": "2026-07-01T14:46:00+09:00", "mjpeg": {"age_sec": 4.668, "exists": true, "mtime": "2026-07-01T14:46:00.683642+09:00", "mtime_epoch": 1782884760.6836421, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195867}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:46:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2716.164, "name": "celery", "pid": 237, "rss_mb": 2372.047, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.003, "pushed": 4.003, "read": 3.97}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.23, "count": 119, "max_ms": 12.669, "p95_ms": 10.336}, "gstreamer_push": {"avg_ms": 0.856, "count": 120, "max_ms": 1.058, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 33.155, "count": 119, "max_ms": 46.418, "p95_ms": 43.389}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.371, "count": 119, "max_ms": 78.803, "p95_ms": 73.818}, "yolo11_640_fire_11class": {"avg_ms": 47.244, "count": 119, "max_ms": 61.716, "p95_ms": 58.729}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.621, "count": 119, "max_ms": 81.936, "p95_ms": 74.893}}, "truncated_count": 0}, "processing_total": {"avg_ms": 203.024, "count": 120, "max_ms": 245.513, "p95_ms": 234.606}, "read_interval": {"avg_ms": 212.893, "count": 119, "max_ms": 255.198, "p95_ms": 245.387}, "triton_wait": {"avg_ms": 61.97, "count": 119, "max_ms": 81.948, "p95_ms": 75.751}, "visualize": {"avg_ms": 9.619, "count": 119, "max_ms": 15.06, "p95_ms": 10.976}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T11:51:05+09:00", "state": "running", "task_id": "86ad926b-8518-43d8-8c13-dc9292a75d1e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144744_d3ef72fd.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 2580.863, "limit_mb": 8796093022207.996}, "detail": {"pid": 272, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:47:44+09:00", "pid": 272, "process": {"exists": true, "fd_count": 5, "hwm_mb": 159.668, "name": "ffmpeg", "pid": 272, "rss_mb": 159.668, "state": "S (sleeping)", "threads": 49}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.895}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "warning", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["mjpeg_age_sec>10s"]}, "instance_id": "d3ef72fd", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 16.597, "exists": true, "mtime": "2026-07-01T14:47:30.001441+09:00", "mtime_epoch": 1782884850.0014405, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259256}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T14:47:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 467.637, "name": "celery", "pid": 267, "rss_mb": 467.637, "state": "R (running)", "threads": 6}, "reason_codes": ["mjpeg_output_stale_warning"], "recommended_action": "degrade_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:47:44+09:00", "state": "running", "task_id": "90a47a32-def6-4dd6-aaf5-9d78ae00be02", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 2770.047, "limit_mb": 8796093022207.996}, "detail": {"pid": 434, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:48:23+09:00", "pid": 434, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.379, "name": "ffmpeg", "pid": 434, "rss_mb": 58.379, "state": "R (running)", "threads": 1}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.943}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 2.624, "exists": true, "mtime": "2026-07-01T14:48:22.458493+09:00", "mtime_epoch": 1782884902.4584928, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254667}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T14:48:25+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 838.477, "name": "celery", "pid": 267, "rss_mb": 745.102, "state": "R (running)", "threads": 10}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3028.297, "limit_mb": 8796093022207.996}, "detail": {"pid": 439, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:48:24+09:00", "pid": 439, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.199, "name": "ffmpeg", "pid": 439, "rss_mb": 136.199, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.902}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 64.024, "exists": true, "mtime": "2026-07-01T14:47:22.741295+09:00", "mtime_epoch": 1782884842.7412946, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192655}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:48:26+09:00", "process": {"exists": true, "fd_count": 106, "hwm_mb": 467.609, "name": "celery", "pid": 237, "rss_mb": 467.609, "state": "R (running)", "threads": 6}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_144836_8bdff53b.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 3794.465, "limit_mb": 8796093022207.996}, "detail": {"pid": 821, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:48:36+09:00", "pid": 821, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.691, "name": "ffmpeg", "pid": 821, "rss_mb": 137.691, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.745}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "8bdff53b", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 7264.645, "exists": true, "mtime": "2026-07-01T12:47:33.440872+09:00", "mtime_epoch": 1782877653.440872, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 262186}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T14:48:38+09:00", "process": {"exists": true, "fd_count": 110, "hwm_mb": 470.695, "name": "celery", "pid": 239, "rss_mb": 470.695, "state": "R (running)", "threads": 7}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:35+09:00", "state": "running", "task_id": "e0c0393c-d1b8-4f5f-a2e9-4985cc08fa8f", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_145016_23ff398f.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 4095.48, "limit_mb": 8796093022207.996}, "detail": {"pid": 1200, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:50:16+09:00", "pid": 1200, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.441, "name": "ffmpeg", "pid": 1200, "rss_mb": 137.441, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.794}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "23ff398f", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.145, "exists": true, "mtime": "2026-07-01T14:50:15.084743+09:00", "mtime_epoch": 1782885015.0847433, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 260282}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T14:50:18+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 694.906, "name": "celery", "pid": 239, "rss_mb": 667.227, "state": "R (running)", "threads": 118}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:50:16+09:00", "state": "running", "task_id": "d6656de0-8852-4493-a409-abd845e4520b", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 3}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3863.441, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "", "last_error_summary": "", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "2026-07-01T14:50:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.077, "last_started_at": "2026-07-01T14:48:24+09:00", "pid": 439, "process": {"exists": false, "pid": 439}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 280.547, "sample_count": 471, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 184.826}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 471, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:50:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:50:19+09:00", "last_frame_read_age_sec": 0.213, "last_frame_read_at": "2026-07-01T14:50:19+09:00", "mjpeg": {"age_sec": 0.205, "exists": true, "mtime": "2026-07-01T14:50:19.636834+09:00", "mtime_epoch": 1782885019.6368341, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206121}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:50:19+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 860.508, "name": "celery", "pid": 237, "rss_mb": 857.496, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.267, "pushed": 4.267, "read": 4.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.495, "count": 128, "max_ms": 14.717, "p95_ms": 10.415}, "gstreamer_push": {"avg_ms": 0.932, "count": 128, "max_ms": 1.985, "p95_ms": 1.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.133, "count": 128, "max_ms": 74.864, "p95_ms": 63.327}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.628, "count": 128, "max_ms": 109.935, "p95_ms": 98.839}, "yolo11_640_fire_11class": {"avg_ms": 54.349, "count": 128, "max_ms": 80.113, "p95_ms": 73.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.02, "count": 128, "max_ms": 101.516, "p95_ms": 95.654}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.078, "count": 128, "max_ms": 318.908, "p95_ms": 271.797}, "read_interval": {"avg_ms": 234.306, "count": 128, "max_ms": 329.35, "p95_ms": 281.823}, "triton_wait": {"avg_ms": 74.377, "count": 128, "max_ms": 109.948, "p95_ms": 100.173}, "visualize": {"avg_ms": 10.238, "count": 128, "max_ms": 15.813, "p95_ms": 12.257}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 3}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3863.434, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "2026-07-01T14:50:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.08, "last_started_at": "2026-07-01T14:48:24+09:00", "pid": 439, "process": {"exists": false, "pid": 439}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 280.547, "sample_count": 471, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 184.823}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 471, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:50:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T14:50:19+09:00", "last_frame_read_age_sec": 0.216, "last_frame_read_at": "2026-07-01T14:50:19+09:00", "mjpeg": {"age_sec": 0.208, "exists": true, "mtime": "2026-07-01T14:50:19.636834+09:00", "mtime_epoch": 1782885019.6368341, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206121}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:50:19+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 860.508, "name": "celery", "pid": 237, "rss_mb": 857.496, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.267, "pushed": 4.267, "read": 4.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.495, "count": 128, "max_ms": 14.717, "p95_ms": 10.415}, "gstreamer_push": {"avg_ms": 0.932, "count": 128, "max_ms": 1.985, "p95_ms": 1.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.133, "count": 128, "max_ms": 74.864, "p95_ms": 63.327}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.628, "count": 128, "max_ms": 109.935, "p95_ms": 98.839}, "yolo11_640_fire_11class": {"avg_ms": 54.349, "count": 128, "max_ms": 80.113, "p95_ms": 73.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.02, "count": 128, "max_ms": 101.516, "p95_ms": 95.654}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.078, "count": 128, "max_ms": 318.908, "p95_ms": 271.797}, "read_interval": {"avg_ms": 234.306, "count": 128, "max_ms": 329.35, "p95_ms": 281.823}, "triton_wait": {"avg_ms": 74.377, "count": 128, "max_ms": 109.948, "p95_ms": 100.173}, "visualize": {"avg_ms": 10.238, "count": 128, "max_ms": 15.813, "p95_ms": 12.257}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 3}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4072.594, "limit_mb": 8796093022207.996}, "detail": {"pid": 1353, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "2026-07-01T14:50:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:50:22+09:00", "pid": 1353, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.715, "name": "ffmpeg", "pid": 1353, "rss_mb": 136.715, "state": "S (sleeping)", "threads": 106}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 280.547, "sample_count": 471, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 179.991}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 471, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:50:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.837, "last_frame_pushed_at": "2026-07-01T14:50:19+09:00", "last_frame_read_age_sec": 5.047, "last_frame_read_at": "2026-07-01T14:50:19+09:00", "mjpeg": {"age_sec": 4.819, "exists": true, "mtime": "2026-07-01T14:50:19.856838+09:00", "mtime_epoch": 1782885019.8568385, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:50:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 860.508, "name": "celery", "pid": 237, "rss_mb": 857.496, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.584, "pushed": 3.584, "read": 3.551}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.482, "count": 107, "max_ms": 14.717, "p95_ms": 10.415}, "gstreamer_push": {"avg_ms": 0.935, "count": 107, "max_ms": 1.985, "p95_ms": 1.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.126, "count": 107, "max_ms": 74.864, "p95_ms": 63.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.119, "count": 107, "max_ms": 109.935, "p95_ms": 99.031}, "yolo11_640_fire_11class": {"avg_ms": 54.063, "count": 107, "max_ms": 80.113, "p95_ms": 69.858}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.09, "count": 107, "max_ms": 101.09, "p95_ms": 95.654}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.552, "count": 107, "max_ms": 318.908, "p95_ms": 274.824}, "read_interval": {"avg_ms": 236.11, "count": 106, "max_ms": 329.35, "p95_ms": 283.831}, "triton_wait": {"avg_ms": 74.807, "count": 107, "max_ms": 109.948, "p95_ms": 100.173}, "visualize": {"avg_ms": 10.266, "count": 107, "max_ms": 15.813, "p95_ms": 12.257}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.855, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_145035_f6f24ba4.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 4075.949, "limit_mb": 8796093022207.996}, "detail": {"pid": 1462, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:50:35+09:00", "pid": 1462, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.156, "name": "ffmpeg", "pid": 1462, "rss_mb": 137.156, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.793}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "f6f24ba4", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.169, "exists": true, "mtime": "2026-07-01T14:50:33.977120+09:00", "mtime_epoch": 1782885033.97712, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 260493}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T14:50:37+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 694.906, "name": "celery", "pid": 239, "rss_mb": 669.309, "state": "R (running)", "threads": 118}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:50:34+09:00", "state": "running", "task_id": "a380140b-728a-4c09-abb0-5bb27acf4a15", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_145103_78813ccd.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 4079.305, "limit_mb": 8796093022207.996}, "detail": {"pid": 1617, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:51:03+09:00", "pid": 1617, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.719, "name": "ffmpeg", "pid": 1617, "rss_mb": 137.719, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.798}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "78813ccd", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 2.881, "exists": true, "mtime": "2026-07-01T14:51:03.001698+09:00", "mtime_epoch": 1782885063.0016978, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 257300}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T14:51:05+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 699.332, "name": "celery", "pid": 239, "rss_mb": 670.34, "state": "R (running)", "threads": 118}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:51:03+09:00", "state": "running", "task_id": "87e8e46d-8219-4840-b691-8c1950acd903", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 8}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3994.953, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b276f33f80] error while decoding MB 71 33, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:52:37+09:00", "last_error_summary": "[h264 @ 0x55b276f33f80] error while decoding MB 71 33, bytestream -11", "last_exit_at": "2026-07-01T14:50:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.05, "last_started_at": "2026-07-01T14:50:22+09:00", "pid": 1353, "process": {"exists": false, "pid": 1353}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.76, "sample_count": 1046, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 46.942}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1046, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:52:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T14:52:37+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T14:52:37+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T14:52:37.683579+09:00", "mtime_epoch": 1782885157.6835792, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208534}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:52:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 860.715, "name": "celery", "pid": 237, "rss_mb": 854.762, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.168, "pushed": 4.168, "read": 4.168}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.609, "count": 124, "max_ms": 12.182, "p95_ms": 10.804}, "gstreamer_push": {"avg_ms": 0.911, "count": 125, "max_ms": 2.95, "p95_ms": 1.002}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.973, "count": 124, "max_ms": 79.389, "p95_ms": 71.005}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.062, "count": 124, "max_ms": 123.814, "p95_ms": 102.771}, "yolo11_640_fire_11class": {"avg_ms": 58.792, "count": 124, "max_ms": 82.564, "p95_ms": 78.845}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.294, "count": 124, "max_ms": 106.185, "p95_ms": 98.59}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.992, "count": 125, "max_ms": 322.545, "p95_ms": 269.05}, "read_interval": {"avg_ms": 241.419, "count": 125, "max_ms": 332.753, "p95_ms": 278.909}, "triton_wait": {"avg_ms": 79.277, "count": 124, "max_ms": 123.828, "p95_ms": 104.849}, "visualize": {"avg_ms": 10.499, "count": 124, "max_ms": 14.394, "p95_ms": 12.279}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 8}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3992.004, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:52:37+09:00", "last_error_summary": "[h264 @ 0x55b276f33f80] error while decoding MB 71 33, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:52:37+09:00", "last_error_summary": "[h264 @ 0x55b276f33f80] error while decoding MB 71 33, bytestream -11", "last_exit_at": "2026-07-01T14:52:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.207, "last_started_at": "2026-07-01T14:50:22+09:00", "pid": 1353, "process": {"exists": false, "pid": 1353}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.76, "sample_count": 1047, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 46.786}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1047, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:52:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:52:37+09:00", "last_frame_read_age_sec": 0.208, "last_frame_read_at": "2026-07-01T14:52:37+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T14:52:37.683579+09:00", "mtime_epoch": 1782885157.6835792, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208534}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:52:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 860.715, "name": "celery", "pid": 237, "rss_mb": 848.57, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.184, "pushed": 4.184, "read": 4.151}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.603, "count": 124, "max_ms": 12.182, "p95_ms": 10.804}, "gstreamer_push": {"avg_ms": 0.92, "count": 125, "max_ms": 2.95, "p95_ms": 1.013}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.931, "count": 124, "max_ms": 79.389, "p95_ms": 71.005}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.748, "count": 124, "max_ms": 123.814, "p95_ms": 102.771}, "yolo11_640_fire_11class": {"avg_ms": 58.711, "count": 124, "max_ms": 82.564, "p95_ms": 78.845}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.182, "count": 124, "max_ms": 106.185, "p95_ms": 98.59}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.715, "count": 125, "max_ms": 322.545, "p95_ms": 269.05}, "read_interval": {"avg_ms": 241.386, "count": 124, "max_ms": 332.753, "p95_ms": 278.909}, "triton_wait": {"avg_ms": 78.963, "count": 124, "max_ms": 123.828, "p95_ms": 104.849}, "visualize": {"avg_ms": 10.495, "count": 124, "max_ms": 14.394, "p95_ms": 12.279}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 8}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3991.941, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:52:37+09:00", "last_error_summary": "[h264 @ 0x55b276f33f80] error while decoding MB 71 33, bytestream -11", "last_exit_at": "2026-07-01T14:52:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.211, "last_started_at": "2026-07-01T14:50:22+09:00", "pid": 1353, "process": {"exists": false, "pid": 1353}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.76, "sample_count": 1047, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 46.783}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1047, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:52:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T14:52:37+09:00", "last_frame_read_age_sec": 0.211, "last_frame_read_at": "2026-07-01T14:52:37+09:00", "mjpeg": {"age_sec": 0.202, "exists": true, "mtime": "2026-07-01T14:52:37.683579+09:00", "mtime_epoch": 1782885157.6835792, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208534}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:52:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 860.715, "name": "celery", "pid": 237, "rss_mb": 848.57, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.184, "pushed": 4.184, "read": 4.15}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.603, "count": 124, "max_ms": 12.182, "p95_ms": 10.804}, "gstreamer_push": {"avg_ms": 0.92, "count": 125, "max_ms": 2.95, "p95_ms": 1.013}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.931, "count": 124, "max_ms": 79.389, "p95_ms": 71.005}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.748, "count": 124, "max_ms": 123.814, "p95_ms": 102.771}, "yolo11_640_fire_11class": {"avg_ms": 58.711, "count": 124, "max_ms": 82.564, "p95_ms": 78.845}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.182, "count": 124, "max_ms": 106.185, "p95_ms": 98.59}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.715, "count": 125, "max_ms": 322.545, "p95_ms": 269.05}, "read_interval": {"avg_ms": 241.386, "count": 124, "max_ms": 332.753, "p95_ms": 278.909}, "triton_wait": {"avg_ms": 78.963, "count": 124, "max_ms": 123.828, "p95_ms": 104.849}, "visualize": {"avg_ms": 10.495, "count": 124, "max_ms": 14.394, "p95_ms": 12.279}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 8}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4098.039, "limit_mb": 8796093022207.996}, "detail": {"pid": 1770, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:52:37+09:00", "last_error_summary": "[h264 @ 0x55b276f33f80] error while decoding MB 71 33, bytestream -11", "last_exit_at": "2026-07-01T14:52:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:52:40+09:00", "pid": 1770, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.828, "name": "ffmpeg", "pid": 1770, "rss_mb": 136.828, "state": "S (sleeping)", "threads": 106}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.76, "sample_count": 1047, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 42.525}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1047, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:52:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.263, "last_frame_pushed_at": "2026-07-01T14:52:37+09:00", "last_frame_read_age_sec": 4.468, "last_frame_read_at": "2026-07-01T14:52:37+09:00", "mjpeg": {"age_sec": 4.247, "exists": true, "mtime": "2026-07-01T14:52:37.895583+09:00", "mtime_epoch": 1782885157.8955834, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204264}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:52:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 860.715, "name": "celery", "pid": 237, "rss_mb": 848.57, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.568, "pushed": 3.568, "read": 3.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.635, "count": 106, "max_ms": 12.182, "p95_ms": 10.817}, "gstreamer_push": {"avg_ms": 0.922, "count": 107, "max_ms": 2.95, "p95_ms": 1.013}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.125, "count": 106, "max_ms": 79.389, "p95_ms": 71.005}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.955, "count": 106, "max_ms": 123.814, "p95_ms": 104.836}, "yolo11_640_fire_11class": {"avg_ms": 58.657, "count": 106, "max_ms": 82.564, "p95_ms": 79.221}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.152, "count": 106, "max_ms": 106.185, "p95_ms": 98.59}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.219, "count": 107, "max_ms": 322.545, "p95_ms": 269.558}, "read_interval": {"avg_ms": 242.978, "count": 106, "max_ms": 332.753, "p95_ms": 279.655}, "triton_wait": {"avg_ms": 79.142, "count": 106, "max_ms": 123.828, "p95_ms": 106.198}, "visualize": {"avg_ms": 10.578, "count": 106, "max_ms": 14.394, "p95_ms": 12.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3940.18, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55658d4ed980] error while decoding MB 2 41, bytestream -9 [in#0/rtsp @ 0x55658d38ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55658d38ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:53:16+09:00", "last_error_summary": "[h264 @ 0x55658d4ed980] error while decoding MB 2 41, bytestream -9 [in#0/rtsp @ 0x55658d38ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55658d38ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 292.966, "last_started_at": "2026-07-01T14:48:23+09:00", "pid": 434, "process": {"exists": false, "pid": 434}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.438, "sample_count": 2480, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 6.977}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2480, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:53:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-01T14:53:15+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T14:53:15+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T14:53:16.000339+09:00", "mtime_epoch": 1782885196.000339, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259034}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T14:53:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 857.445, "name": "celery", "pid": 267, "rss_mb": 851.906, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.737, "pushed": 0.737, "read": 0.794}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.567, "count": 13, "max_ms": 2.098, "p95_ms": 2.098}, "gstreamer_push": {"avg_ms": 1.541, "count": 13, "max_ms": 3.843, "p95_ms": 3.843}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.665, "count": 13, "max_ms": 62.218, "p95_ms": 62.218}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.607, "count": 13, "max_ms": 87.279, "p95_ms": 87.279}, "yolo11_640_fire_11class": {"avg_ms": 53.862, "count": 13, "max_ms": 63.761, "p95_ms": 63.761}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.01, "count": 13, "max_ms": 86.707, "p95_ms": 86.707}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.878, "count": 13, "max_ms": 97.505, "p95_ms": 97.505}, "read_interval": {"avg_ms": 2460.194, "count": 14, "max_ms": 16866.975, "p95_ms": 16866.975}, "triton_wait": {"avg_ms": 62.668, "count": 13, "max_ms": 87.289, "p95_ms": 87.289}, "visualize": {"avg_ms": 3.7, "count": 13, "max_ms": 4.54, "p95_ms": 4.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.631, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3939.398, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:53:16+09:00", "last_error_summary": "[h264 @ 0x55658d4ed980] error while decoding MB 2 41, bytestream -9 [in#0/rtsp @ 0x55658d38ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55658d38ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:53:16+09:00", "last_error_summary": "[h264 @ 0x55658d4ed980] error while decoding MB 2 41, bytestream -9 [in#0/rtsp @ 0x55658d38ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55658d38ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T14:53:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 293.016, "last_started_at": "2026-07-01T14:48:23+09:00", "pid": 434, "process": {"exists": false, "pid": 434}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.438, "sample_count": 2481, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 6.928}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2481, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:53:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T14:53:16+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-01T14:53:15+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-01T14:53:16.000339+09:00", "mtime_epoch": 1782885196.000339, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259034}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T14:53:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 857.445, "name": "celery", "pid": 267, "rss_mb": 851.945, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.792, "pushed": 0.792, "read": 0.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.562, "count": 14, "max_ms": 2.098, "p95_ms": 2.098}, "gstreamer_push": {"avg_ms": 1.524, "count": 14, "max_ms": 3.843, "p95_ms": 3.843}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.81, "count": 14, "max_ms": 62.218, "p95_ms": 62.218}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.145, "count": 14, "max_ms": 87.279, "p95_ms": 87.279}, "yolo11_640_fire_11class": {"avg_ms": 54.123, "count": 14, "max_ms": 63.761, "p95_ms": 63.761}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.118, "count": 14, "max_ms": 86.707, "p95_ms": 86.707}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.38, "count": 14, "max_ms": 97.505, "p95_ms": 97.505}, "read_interval": {"avg_ms": 2460.194, "count": 14, "max_ms": 16866.975, "p95_ms": 16866.975}, "triton_wait": {"avg_ms": 64.202, "count": 14, "max_ms": 87.289, "p95_ms": 87.289}, "visualize": {"avg_ms": 3.687, "count": 14, "max_ms": 4.54, "p95_ms": 4.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.681, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3939.391, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:53:16+09:00", "last_error_summary": "[h264 @ 0x55658d4ed980] error while decoding MB 2 41, bytestream -9 [in#0/rtsp @ 0x55658d38ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55658d38ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T14:53:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 293.023, "last_started_at": "2026-07-01T14:48:23+09:00", "pid": 434, "process": {"exists": false, "pid": 434}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.438, "sample_count": 2481, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 6.921}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2481, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:53:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T14:53:16+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-01T14:53:15+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-01T14:53:16.000339+09:00", "mtime_epoch": 1782885196.000339, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259034}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T14:53:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 857.445, "name": "celery", "pid": 267, "rss_mb": 851.945, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.792, "pushed": 0.792, "read": 0.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.562, "count": 14, "max_ms": 2.098, "p95_ms": 2.098}, "gstreamer_push": {"avg_ms": 1.524, "count": 14, "max_ms": 3.843, "p95_ms": 3.843}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.81, "count": 14, "max_ms": 62.218, "p95_ms": 62.218}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.145, "count": 14, "max_ms": 87.279, "p95_ms": 87.279}, "yolo11_640_fire_11class": {"avg_ms": 54.123, "count": 14, "max_ms": 63.761, "p95_ms": 63.761}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.118, "count": 14, "max_ms": 86.707, "p95_ms": 86.707}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.38, "count": 14, "max_ms": 97.505, "p95_ms": 97.505}, "read_interval": {"avg_ms": 2460.194, "count": 14, "max_ms": 16866.975, "p95_ms": 16866.975}, "triton_wait": {"avg_ms": 64.202, "count": 14, "max_ms": 87.289, "p95_ms": 87.289}, "visualize": {"avg_ms": 3.687, "count": 14, "max_ms": 4.54, "p95_ms": 4.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.687, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4102.77, "limit_mb": 8796093022207.996}, "detail": {"pid": 1877, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:53:16+09:00", "last_error_summary": "[h264 @ 0x55658d4ed980] error while decoding MB 2 41, bytestream -9 [in#0/rtsp @ 0x55658d38ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55658d38ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T14:53:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:53:18+09:00", "pid": 1877, "process": {"exists": true, "fd_count": 5, "hwm_mb": 194.875, "name": "ffmpeg", "pid": 1877, "rss_mb": 194.875, "state": "S (sleeping)", "threads": 106}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.438, "sample_count": 2481, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 2.198}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2481, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:53:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.738, "last_frame_pushed_at": "2026-07-01T14:53:16+09:00", "last_frame_read_age_sec": 4.834, "last_frame_read_at": "2026-07-01T14:53:15+09:00", "mjpeg": {"age_sec": 4.722, "exists": true, "mtime": "2026-07-01T14:53:16.104341+09:00", "mtime_epoch": 1782885196.104341, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 255684}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T14:53:20+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 857.445, "name": "celery", "pid": 267, "rss_mb": 851.945, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.625, "pushed": 0.625, "read": 0.625}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.562, "count": 14, "max_ms": 2.098, "p95_ms": 2.098}, "gstreamer_push": {"avg_ms": 1.524, "count": 14, "max_ms": 3.843, "p95_ms": 3.843}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.81, "count": 14, "max_ms": 62.218, "p95_ms": 62.218}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.145, "count": 14, "max_ms": 87.279, "p95_ms": 87.279}, "yolo11_640_fire_11class": {"avg_ms": 54.123, "count": 14, "max_ms": 63.761, "p95_ms": 63.761}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.118, "count": 14, "max_ms": 86.707, "p95_ms": 86.707}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.38, "count": 14, "max_ms": 97.505, "p95_ms": 97.505}, "read_interval": {"avg_ms": 2460.194, "count": 14, "max_ms": 16866.975, "p95_ms": 16866.975}, "triton_wait": {"avg_ms": 64.202, "count": 14, "max_ms": 87.289, "p95_ms": 87.289}, "visualize": {"avg_ms": 3.687, "count": 14, "max_ms": 4.54, "p95_ms": 4.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.41, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 12}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4040.082, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564081fafc40] error while decoding MB 72 40, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:54:29+09:00", "last_error_summary": "[h264 @ 0x564081fafc40] error while decoding MB 72 40, bytestream -5", "last_exit_at": "2026-07-01T14:52:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.141, "last_started_at": "2026-07-01T14:52:40+09:00", "pid": 1770, "process": {"exists": false, "pid": 1770}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.963, "sample_count": 1253, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1508, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:54:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-01T14:54:29+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-01T14:54:29+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T14:54:29.237789+09:00", "mtime_epoch": 1782885269.2377892, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209962}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:54:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 860.715, "name": "celery", "pid": 237, "rss_mb": 857.781, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.084, "pushed": 4.084, "read": 4.084}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.745, "count": 122, "max_ms": 13.808, "p95_ms": 10.916}, "gstreamer_push": {"avg_ms": 0.964, "count": 122, "max_ms": 2.965, "p95_ms": 1.306}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.847, "count": 122, "max_ms": 79.321, "p95_ms": 71.742}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.53, "count": 122, "max_ms": 116.99, "p95_ms": 108.141}, "yolo11_640_fire_11class": {"avg_ms": 57.675, "count": 122, "max_ms": 87.621, "p95_ms": 75.956}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.239, "count": 122, "max_ms": 111.799, "p95_ms": 96.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.307, "count": 122, "max_ms": 319.007, "p95_ms": 275.46}, "read_interval": {"avg_ms": 246.172, "count": 122, "max_ms": 330.429, "p95_ms": 286.41}, "triton_wait": {"avg_ms": 78.981, "count": 122, "max_ms": 117.002, "p95_ms": 108.154}, "visualize": {"avg_ms": 10.661, "count": 122, "max_ms": 14.706, "p95_ms": 12.786}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 12}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4026.715, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:54:29+09:00", "last_error_summary": "[h264 @ 0x564081fafc40] error while decoding MB 72 40, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:54:29+09:00", "last_error_summary": "[h264 @ 0x564081fafc40] error while decoding MB 72 40, bytestream -5", "last_exit_at": "2026-07-01T14:54:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.301, "last_started_at": "2026-07-01T14:52:40+09:00", "pid": 1770, "process": {"exists": false, "pid": 1770}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.963, "sample_count": 1253, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1509, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:54:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:54:29+09:00", "last_frame_read_age_sec": 0.216, "last_frame_read_at": "2026-07-01T14:54:29+09:00", "mjpeg": {"age_sec": 0.205, "exists": true, "mtime": "2026-07-01T14:54:29.237789+09:00", "mtime_epoch": 1782885269.2377892, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209962}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:54:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 860.715, "name": "celery", "pid": 237, "rss_mb": 857.785, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.088, "pushed": 4.088, "read": 4.055}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.779, "count": 122, "max_ms": 13.808, "p95_ms": 11.035}, "gstreamer_push": {"avg_ms": 0.964, "count": 122, "max_ms": 2.965, "p95_ms": 1.306}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.671, "count": 122, "max_ms": 79.321, "p95_ms": 71.742}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.267, "count": 122, "max_ms": 116.99, "p95_ms": 108.141}, "yolo11_640_fire_11class": {"avg_ms": 57.467, "count": 122, "max_ms": 87.621, "p95_ms": 75.956}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.977, "count": 122, "max_ms": 111.799, "p95_ms": 96.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.269, "count": 122, "max_ms": 319.007, "p95_ms": 275.46}, "read_interval": {"avg_ms": 246.355, "count": 121, "max_ms": 330.429, "p95_ms": 286.41}, "triton_wait": {"avg_ms": 78.718, "count": 122, "max_ms": 117.002, "p95_ms": 108.154}, "visualize": {"avg_ms": 10.676, "count": 122, "max_ms": 14.706, "p95_ms": 12.786}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.841, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 12}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4027.582, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:54:29+09:00", "last_error_summary": "[h264 @ 0x564081fafc40] error while decoding MB 72 40, bytestream -5", "last_exit_at": "2026-07-01T14:54:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.305, "last_started_at": "2026-07-01T14:52:40+09:00", "pid": 1770, "process": {"exists": false, "pid": 1770}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.963, "sample_count": 1253, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1509, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:54:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T14:54:29+09:00", "last_frame_read_age_sec": 0.219, "last_frame_read_at": "2026-07-01T14:54:29+09:00", "mjpeg": {"age_sec": 0.208, "exists": true, "mtime": "2026-07-01T14:54:29.237789+09:00", "mtime_epoch": 1782885269.2377892, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209962}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:54:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 860.715, "name": "celery", "pid": 237, "rss_mb": 857.785, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.088, "pushed": 4.088, "read": 4.054}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.779, "count": 122, "max_ms": 13.808, "p95_ms": 11.035}, "gstreamer_push": {"avg_ms": 0.964, "count": 122, "max_ms": 2.965, "p95_ms": 1.306}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.671, "count": 122, "max_ms": 79.321, "p95_ms": 71.742}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.267, "count": 122, "max_ms": 116.99, "p95_ms": 108.141}, "yolo11_640_fire_11class": {"avg_ms": 57.467, "count": 122, "max_ms": 87.621, "p95_ms": 75.956}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.977, "count": 122, "max_ms": 111.799, "p95_ms": 96.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.269, "count": 122, "max_ms": 319.007, "p95_ms": 275.46}, "read_interval": {"avg_ms": 246.355, "count": 121, "max_ms": 330.429, "p95_ms": 286.41}, "triton_wait": {"avg_ms": 78.718, "count": 122, "max_ms": 117.002, "p95_ms": 108.154}, "visualize": {"avg_ms": 10.676, "count": 122, "max_ms": 14.706, "p95_ms": 12.786}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.844, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 12}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4158.09, "limit_mb": 8796093022207.996}, "detail": {"pid": 2029, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:54:29+09:00", "last_error_summary": "[h264 @ 0x564081fafc40] error while decoding MB 72 40, bytestream -5", "last_exit_at": "2026-07-01T14:54:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:54:31+09:00", "pid": 2029, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.73, "name": "ffmpeg", "pid": 2029, "rss_mb": 144.73, "state": "S (sleeping)", "threads": 106}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 265.578, "sample_count": 1236, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1509, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:54:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.151, "last_frame_pushed_at": "2026-07-01T14:54:29+09:00", "last_frame_read_age_sec": 4.364, "last_frame_read_at": "2026-07-01T14:54:29+09:00", "mjpeg": {"age_sec": 4.133, "exists": true, "mtime": "2026-07-01T14:54:29.457794+09:00", "mtime_epoch": 1782885269.4577937, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202177}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:54:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 860.715, "name": "celery", "pid": 237, "rss_mb": 857.785, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.483, "pushed": 3.483, "read": 3.45}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.846, "count": 104, "max_ms": 13.808, "p95_ms": 11.035}, "gstreamer_push": {"avg_ms": 0.964, "count": 104, "max_ms": 2.965, "p95_ms": 1.17}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.249, "count": 104, "max_ms": 78.544, "p95_ms": 70.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.64, "count": 104, "max_ms": 116.99, "p95_ms": 108.141}, "yolo11_640_fire_11class": {"avg_ms": 57.605, "count": 104, "max_ms": 87.621, "p95_ms": 75.956}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.21, "count": 104, "max_ms": 111.799, "p95_ms": 94.086}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.393, "count": 104, "max_ms": 319.007, "p95_ms": 275.657}, "read_interval": {"avg_ms": 249.577, "count": 103, "max_ms": 330.429, "p95_ms": 286.931}, "triton_wait": {"avg_ms": 79.068, "count": 104, "max_ms": 117.002, "p95_ms": 108.154}, "visualize": {"avg_ms": 10.818, "count": 104, "max_ms": 14.706, "p95_ms": 12.789}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.859, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 16}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4042.996, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5643a79159c0] error while decoding MB 51 6, bytestream -18", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:56:20+09:00", "last_error_summary": "[h264 @ 0x5643a79159c0] error while decoding MB 51 6, bytestream -18", "last_exit_at": "2026-07-01T14:54:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.392, "last_started_at": "2026-07-01T14:54:31+09:00", "pid": 2029, "process": {"exists": false, "pid": 2029}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.328, "sample_count": 1244, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1957, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:56:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.051, "last_frame_pushed_at": "2026-07-01T14:56:20+09:00", "last_frame_read_age_sec": 0.044, "last_frame_read_at": "2026-07-01T14:56:20+09:00", "mjpeg": {"age_sec": 0.031, "exists": true, "mtime": "2026-07-01T14:56:20.950400+09:00", "mtime_epoch": 1782885380.9504004, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:56:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.242, "name": "celery", "pid": 237, "rss_mb": 855.52, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.221, "pushed": 4.221, "read": 4.221}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.262, "count": 126, "max_ms": 12.496, "p95_ms": 10.485}, "gstreamer_push": {"avg_ms": 0.951, "count": 126, "max_ms": 3.049, "p95_ms": 1.844}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.189, "count": 126, "max_ms": 82.43, "p95_ms": 75.792}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.997, "count": 126, "max_ms": 124.794, "p95_ms": 108.098}, "yolo11_640_fire_11class": {"avg_ms": 58.628, "count": 126, "max_ms": 85.515, "p95_ms": 79.63}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.059, "count": 126, "max_ms": 108.042, "p95_ms": 101.702}}, "truncated_count": 0}, "processing_total": {"avg_ms": 228.252, "count": 126, "max_ms": 297.883, "p95_ms": 268.525}, "read_interval": {"avg_ms": 238.124, "count": 126, "max_ms": 308.379, "p95_ms": 279.125}, "triton_wait": {"avg_ms": 80.872, "count": 126, "max_ms": 124.806, "p95_ms": 108.11}, "visualize": {"avg_ms": 10.049, "count": 126, "max_ms": 14.531, "p95_ms": 12.678}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.848, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 16}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4042.23, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:56:20+09:00", "last_error_summary": "[h264 @ 0x5643a79159c0] error while decoding MB 51 6, bytestream -18", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:56:20+09:00", "last_error_summary": "[h264 @ 0x5643a79159c0] error while decoding MB 51 6, bytestream -18", "last_exit_at": "2026-07-01T14:56:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.589, "last_started_at": "2026-07-01T14:54:31+09:00", "pid": 2029, "process": {"exists": false, "pid": 2029}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.328, "sample_count": 1244, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1958, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:56:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T14:56:21+09:00", "last_frame_read_age_sec": 0.241, "last_frame_read_at": "2026-07-01T14:56:20+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T14:56:20.950400+09:00", "mtime_epoch": 1782885380.9504004, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:56:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.242, "name": "celery", "pid": 237, "rss_mb": 855.523, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.224, "pushed": 4.224, "read": 4.191}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.261, "count": 126, "max_ms": 12.496, "p95_ms": 10.485}, "gstreamer_push": {"avg_ms": 0.951, "count": 126, "max_ms": 3.049, "p95_ms": 1.844}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.37, "count": 126, "max_ms": 82.43, "p95_ms": 75.792}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.212, "count": 126, "max_ms": 124.794, "p95_ms": 108.098}, "yolo11_640_fire_11class": {"avg_ms": 58.775, "count": 126, "max_ms": 85.515, "p95_ms": 79.63}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.306, "count": 126, "max_ms": 108.042, "p95_ms": 101.702}}, "truncated_count": 0}, "processing_total": {"avg_ms": 228.54, "count": 126, "max_ms": 297.883, "p95_ms": 268.525}, "read_interval": {"avg_ms": 238.354, "count": 125, "max_ms": 308.379, "p95_ms": 279.125}, "triton_wait": {"avg_ms": 81.12, "count": 126, "max_ms": 124.806, "p95_ms": 108.11}, "visualize": {"avg_ms": 10.054, "count": 126, "max_ms": 14.531, "p95_ms": 12.678}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.828, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 16}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4042.188, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:56:20+09:00", "last_error_summary": "[h264 @ 0x5643a79159c0] error while decoding MB 51 6, bytestream -18", "last_exit_at": "2026-07-01T14:56:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.594, "last_started_at": "2026-07-01T14:54:31+09:00", "pid": 2029, "process": {"exists": false, "pid": 2029}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.328, "sample_count": 1244, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1958, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:56:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T14:56:21+09:00", "last_frame_read_age_sec": 0.246, "last_frame_read_at": "2026-07-01T14:56:20+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-01T14:56:20.950400+09:00", "mtime_epoch": 1782885380.9504004, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:56:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.242, "name": "celery", "pid": 237, "rss_mb": 855.523, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.224, "pushed": 4.224, "read": 4.19}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.261, "count": 126, "max_ms": 12.496, "p95_ms": 10.485}, "gstreamer_push": {"avg_ms": 0.951, "count": 126, "max_ms": 3.049, "p95_ms": 1.844}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.37, "count": 126, "max_ms": 82.43, "p95_ms": 75.792}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.212, "count": 126, "max_ms": 124.794, "p95_ms": 108.098}, "yolo11_640_fire_11class": {"avg_ms": 58.775, "count": 126, "max_ms": 85.515, "p95_ms": 79.63}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.306, "count": 126, "max_ms": 108.042, "p95_ms": 101.702}}, "truncated_count": 0}, "processing_total": {"avg_ms": 228.54, "count": 126, "max_ms": 297.883, "p95_ms": 268.525}, "read_interval": {"avg_ms": 238.354, "count": 125, "max_ms": 308.379, "p95_ms": 279.125}, "triton_wait": {"avg_ms": 81.12, "count": 126, "max_ms": 124.806, "p95_ms": 108.11}, "visualize": {"avg_ms": 10.054, "count": 126, "max_ms": 14.531, "p95_ms": 12.678}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.833, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 16}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4128.715, "limit_mb": 8796093022207.996}, "detail": {"pid": 2136, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:56:20+09:00", "last_error_summary": "[h264 @ 0x5643a79159c0] error while decoding MB 51 6, bytestream -18", "last_exit_at": "2026-07-01T14:56:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:56:24+09:00", "pid": 2136, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.695, "name": "ffmpeg", "pid": 2136, "rss_mb": 133.695, "state": "S (sleeping)", "threads": 106}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.153, "sample_count": 1222, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 1958, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:56:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.965, "last_frame_pushed_at": "2026-07-01T14:56:21+09:00", "last_frame_read_age_sec": 5.202, "last_frame_read_at": "2026-07-01T14:56:20+09:00", "mjpeg": {"age_sec": 4.95, "exists": true, "mtime": "2026-07-01T14:56:21.190400+09:00", "mtime_epoch": 1782885381.1904, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201606}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:56:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.242, "name": "celery", "pid": 237, "rss_mb": 855.523, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.552, "pushed": 3.552, "read": 3.518}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.35, "count": 106, "max_ms": 12.496, "p95_ms": 10.485}, "gstreamer_push": {"avg_ms": 0.955, "count": 106, "max_ms": 3.049, "p95_ms": 1.844}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.431, "count": 106, "max_ms": 82.43, "p95_ms": 75.792}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.562, "count": 106, "max_ms": 124.794, "p95_ms": 104.262}, "yolo11_640_fire_11class": {"avg_ms": 58.649, "count": 106, "max_ms": 81.893, "p95_ms": 79.341}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.802, "count": 106, "max_ms": 106.966, "p95_ms": 100.756}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.187, "count": 106, "max_ms": 296.385, "p95_ms": 264.153}, "read_interval": {"avg_ms": 237.003, "count": 105, "max_ms": 307.783, "p95_ms": 274.549}, "triton_wait": {"avg_ms": 80.545, "count": 106, "max_ms": 124.806, "p95_ms": 104.275}, "visualize": {"avg_ms": 10.181, "count": 106, "max_ms": 14.059, "p95_ms": 12.678}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4036.34, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55babb189ac0] error while decoding MB 75 39, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:58:15+09:00", "last_error_summary": "[h264 @ 0x55babb189ac0] error while decoding MB 75 39, bytestream -6", "last_exit_at": "2026-07-01T14:56:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.109, "last_started_at": "2026-07-01T14:56:24+09:00", "pid": 2136, "process": {"exists": false, "pid": 2136}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.641, "sample_count": 1226, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2418, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:58:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T14:58:15+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T14:58:15+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T14:58:15.206440+09:00", "mtime_epoch": 1782885495.2064402, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206132}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:58:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 850.371, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.16, "pushed": 4.16, "read": 4.16}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.979, "count": 124, "max_ms": 14.884, "p95_ms": 11.253}, "gstreamer_push": {"avg_ms": 0.98, "count": 124, "max_ms": 2.995, "p95_ms": 1.817}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.365, "count": 124, "max_ms": 79.019, "p95_ms": 69.855}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.189, "count": 124, "max_ms": 109.825, "p95_ms": 100.629}, "yolo11_640_fire_11class": {"avg_ms": 55.017, "count": 124, "max_ms": 81.738, "p95_ms": 73.852}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.752, "count": 124, "max_ms": 105.503, "p95_ms": 98.996}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.918, "count": 124, "max_ms": 329.388, "p95_ms": 267.733}, "read_interval": {"avg_ms": 241.79, "count": 124, "max_ms": 340.922, "p95_ms": 278.609}, "triton_wait": {"avg_ms": 75.773, "count": 124, "max_ms": 109.839, "p95_ms": 101.749}, "visualize": {"avg_ms": 10.452, "count": 124, "max_ms": 15.307, "p95_ms": 12.232}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.806, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4041.582, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:58:15+09:00", "last_error_summary": "[h264 @ 0x55babb189ac0] error while decoding MB 75 39, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:58:15+09:00", "last_error_summary": "[h264 @ 0x55babb189ac0] error while decoding MB 75 39, bytestream -6", "last_exit_at": "2026-07-01T14:58:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.301, "last_started_at": "2026-07-01T14:56:24+09:00", "pid": 2136, "process": {"exists": false, "pid": 2136}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.641, "sample_count": 1226, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2419, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:58:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T14:58:15+09:00", "last_frame_read_age_sec": 0.243, "last_frame_read_at": "2026-07-01T14:58:15+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-01T14:58:15.206440+09:00", "mtime_epoch": 1782885495.2064402, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206132}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:58:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.562, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.167, "pushed": 4.167, "read": 4.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.981, "count": 124, "max_ms": 14.884, "p95_ms": 11.253}, "gstreamer_push": {"avg_ms": 0.979, "count": 125, "max_ms": 2.995, "p95_ms": 1.817}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.336, "count": 124, "max_ms": 79.019, "p95_ms": 69.855}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.407, "count": 124, "max_ms": 109.825, "p95_ms": 100.629}, "yolo11_640_fire_11class": {"avg_ms": 55.064, "count": 124, "max_ms": 81.738, "p95_ms": 73.852}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.774, "count": 124, "max_ms": 105.503, "p95_ms": 98.996}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.969, "count": 125, "max_ms": 329.388, "p95_ms": 267.733}, "read_interval": {"avg_ms": 241.79, "count": 124, "max_ms": 340.922, "p95_ms": 278.609}, "triton_wait": {"avg_ms": 75.991, "count": 124, "max_ms": 109.839, "p95_ms": 101.749}, "visualize": {"avg_ms": 10.467, "count": 124, "max_ms": 15.307, "p95_ms": 12.232}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4041.574, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:58:15+09:00", "last_error_summary": "[h264 @ 0x55babb189ac0] error while decoding MB 75 39, bytestream -6", "last_exit_at": "2026-07-01T14:58:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.304, "last_started_at": "2026-07-01T14:56:24+09:00", "pid": 2136, "process": {"exists": false, "pid": 2136}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.641, "sample_count": 1226, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2419, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:58:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T14:58:15+09:00", "last_frame_read_age_sec": 0.246, "last_frame_read_at": "2026-07-01T14:58:15+09:00", "mjpeg": {"age_sec": 0.236, "exists": true, "mtime": "2026-07-01T14:58:15.206440+09:00", "mtime_epoch": 1782885495.2064402, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206132}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:58:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.562, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.167, "pushed": 4.167, "read": 4.133}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.981, "count": 124, "max_ms": 14.884, "p95_ms": 11.253}, "gstreamer_push": {"avg_ms": 0.979, "count": 125, "max_ms": 2.995, "p95_ms": 1.817}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.336, "count": 124, "max_ms": 79.019, "p95_ms": 69.855}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.407, "count": 124, "max_ms": 109.825, "p95_ms": 100.629}, "yolo11_640_fire_11class": {"avg_ms": 55.064, "count": 124, "max_ms": 81.738, "p95_ms": 73.852}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.774, "count": 124, "max_ms": 105.503, "p95_ms": 98.996}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.969, "count": 125, "max_ms": 329.388, "p95_ms": 267.733}, "read_interval": {"avg_ms": 241.79, "count": 124, "max_ms": 340.922, "p95_ms": 278.609}, "triton_wait": {"avg_ms": 75.991, "count": 124, "max_ms": 109.839, "p95_ms": 101.749}, "visualize": {"avg_ms": 10.467, "count": 124, "max_ms": 15.307, "p95_ms": 12.232}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 20}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4149.246, "limit_mb": 8796093022207.996}, "detail": {"pid": 2243, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:58:15+09:00", "last_error_summary": "[h264 @ 0x55babb189ac0] error while decoding MB 75 39, bytestream -6", "last_exit_at": "2026-07-01T14:58:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:58:18+09:00", "pid": 2243, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.625, "name": "ffmpeg", "pid": 2243, "rss_mb": 138.625, "state": "S (sleeping)", "threads": 106}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.672, "sample_count": 1204, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2419, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:58:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.954, "last_frame_pushed_at": "2026-07-01T14:58:15+09:00", "last_frame_read_age_sec": 5.193, "last_frame_read_at": "2026-07-01T14:58:15+09:00", "mjpeg": {"age_sec": 4.936, "exists": true, "mtime": "2026-07-01T14:58:15.454441+09:00", "mtime_epoch": 1782885495.4544408, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206278}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T14:58:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.562, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.487, "pushed": 3.487, "read": 3.454}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.008, "count": 104, "max_ms": 14.884, "p95_ms": 11.253}, "gstreamer_push": {"avg_ms": 0.985, "count": 104, "max_ms": 2.995, "p95_ms": 1.817}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.059, "count": 104, "max_ms": 77.744, "p95_ms": 69.855}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.725, "count": 104, "max_ms": 109.825, "p95_ms": 100.027}, "yolo11_640_fire_11class": {"avg_ms": 54.905, "count": 104, "max_ms": 81.738, "p95_ms": 73.558}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.015, "count": 104, "max_ms": 105.503, "p95_ms": 98.204}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.329, "count": 104, "max_ms": 329.388, "p95_ms": 270.075}, "read_interval": {"avg_ms": 241.155, "count": 103, "max_ms": 340.922, "p95_ms": 279.412}, "triton_wait": {"avg_ms": 75.428, "count": 104, "max_ms": 109.839, "p95_ms": 103.848}, "visualize": {"avg_ms": 10.49, "count": 104, "max_ms": 15.307, "p95_ms": 12.232}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.823, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 48}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3975.219, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e1e0617cc0] error while decoding MB 84 33, bytestream -7 [in#0/rtsp @ 0x55e1dfaa4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1dfaa4e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:59:33+09:00", "last_error_summary": "[h264 @ 0x55e1e0617cc0] error while decoding MB 84 33, bytestream -7 [in#0/rtsp @ 0x55e1dfaa4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1dfaa4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T14:53:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 375.12, "last_started_at": "2026-07-01T14:53:18+09:00", "pid": 1877, "process": {"exists": false, "pid": 1877}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.681, "sample_count": 2571, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5793, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:59:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-01T14:59:33+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-01T14:59:33+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-01T14:59:33.890678+09:00", "mtime_epoch": 1782885573.8906782, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242232}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T14:59:33+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 860.055, "name": "celery", "pid": 267, "rss_mb": 856.203, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 16, "pushed_frames": 16, "read_frames": 16, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.534, "pushed": 0.534, "read": 0.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.649, "count": 17, "max_ms": 2.345, "p95_ms": 2.345}, "gstreamer_push": {"avg_ms": 1.343, "count": 16, "max_ms": 3.571, "p95_ms": 3.571}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.12, "count": 17, "max_ms": 52.799, "p95_ms": 52.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.573, "count": 17, "max_ms": 111.235, "p95_ms": 111.235}, "yolo11_640_fire_11class": {"avg_ms": 55.469, "count": 17, "max_ms": 82.801, "p95_ms": 82.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.467, "count": 17, "max_ms": 71.695, "p95_ms": 71.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.006, "count": 16, "max_ms": 121.522, "p95_ms": 121.522}, "read_interval": {"avg_ms": 1874.518, "count": 16, "max_ms": 15988.715, "p95_ms": 15988.715}, "triton_wait": {"avg_ms": 66.585, "count": 17, "max_ms": 111.249, "p95_ms": 111.249}, "visualize": {"avg_ms": 3.42, "count": 17, "max_ms": 4.761, "p95_ms": 4.761}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 48}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3974.859, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:59:33+09:00", "last_error_summary": "[h264 @ 0x55e1e0617cc0] error while decoding MB 84 33, bytestream -7 [in#0/rtsp @ 0x55e1dfaa4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1dfaa4e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:59:33+09:00", "last_error_summary": "[h264 @ 0x55e1e0617cc0] error while decoding MB 84 33, bytestream -7 [in#0/rtsp @ 0x55e1dfaa4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1dfaa4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T14:59:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 375.141, "last_started_at": "2026-07-01T14:53:18+09:00", "pid": 1877, "process": {"exists": false, "pid": 1877}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.575, "sample_count": 2572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5793, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:59:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T14:59:33+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-01T14:59:33+09:00", "mjpeg": {"age_sec": 0.075, "exists": true, "mtime": "2026-07-01T14:59:33.890678+09:00", "mtime_epoch": 1782885573.8906782, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242232}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T14:59:33+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 860.055, "name": "celery", "pid": 267, "rss_mb": 856.207, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 16, "pushed_frames": 16, "read_frames": 16, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.533, "pushed": 0.533, "read": 0.533}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.657, "count": 16, "max_ms": 2.345, "p95_ms": 2.345}, "gstreamer_push": {"avg_ms": 1.344, "count": 16, "max_ms": 3.571, "p95_ms": 3.571}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.19, "count": 16, "max_ms": 52.799, "p95_ms": 52.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.215, "count": 16, "max_ms": 111.235, "p95_ms": 111.235}, "yolo11_640_fire_11class": {"avg_ms": 55.812, "count": 16, "max_ms": 82.801, "p95_ms": 82.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.71, "count": 16, "max_ms": 71.695, "p95_ms": 71.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.177, "count": 16, "max_ms": 121.522, "p95_ms": 121.522}, "read_interval": {"avg_ms": 1874.518, "count": 16, "max_ms": 15988.715, "p95_ms": 15988.715}, "triton_wait": {"avg_ms": 67.227, "count": 16, "max_ms": 111.249, "p95_ms": 111.249}, "visualize": {"avg_ms": 3.423, "count": 16, "max_ms": 4.761, "p95_ms": 4.761}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 48}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3978.441, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:59:33+09:00", "last_error_summary": "[h264 @ 0x55e1e0617cc0] error while decoding MB 84 33, bytestream -7 [in#0/rtsp @ 0x55e1dfaa4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1dfaa4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T14:59:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 375.147, "last_started_at": "2026-07-01T14:53:18+09:00", "pid": 1877, "process": {"exists": false, "pid": 1877}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.575, "sample_count": 2572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5793, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:59:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-01T14:59:33+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-01T14:59:33+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-01T14:59:33.890678+09:00", "mtime_epoch": 1782885573.8906782, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242232}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T14:59:33+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 860.055, "name": "celery", "pid": 267, "rss_mb": 856.207, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 16, "pushed_frames": 16, "read_frames": 16, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.533, "pushed": 0.533, "read": 0.533}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.657, "count": 16, "max_ms": 2.345, "p95_ms": 2.345}, "gstreamer_push": {"avg_ms": 1.344, "count": 16, "max_ms": 3.571, "p95_ms": 3.571}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.19, "count": 16, "max_ms": 52.799, "p95_ms": 52.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.215, "count": 16, "max_ms": 111.235, "p95_ms": 111.235}, "yolo11_640_fire_11class": {"avg_ms": 55.812, "count": 16, "max_ms": 82.801, "p95_ms": 82.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.71, "count": 16, "max_ms": 71.695, "p95_ms": 71.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.177, "count": 16, "max_ms": 121.522, "p95_ms": 121.522}, "read_interval": {"avg_ms": 1874.518, "count": 16, "max_ms": 15988.715, "p95_ms": 15988.715}, "triton_wait": {"avg_ms": 67.227, "count": 16, "max_ms": 111.249, "p95_ms": 111.249}, "visualize": {"avg_ms": 3.423, "count": 16, "max_ms": 4.761, "p95_ms": 4.761}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 48}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3999.965, "limit_mb": 8796093022207.996}, "detail": {"pid": 2350, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:59:33+09:00", "last_error_summary": "[h264 @ 0x55e1e0617cc0] error while decoding MB 84 33, bytestream -7 [in#0/rtsp @ 0x55e1dfaa4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e1dfaa4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T14:59:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T14:59:36+09:00", "pid": 2350, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.922, "name": "ffmpeg", "pid": 2350, "rss_mb": 57.922, "state": "S (sleeping)", "threads": 1}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.84, "sample_count": 2532, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5793, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T14:59:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.162, "last_frame_pushed_at": "2026-07-01T14:59:33+09:00", "last_frame_read_age_sec": 4.233, "last_frame_read_at": "2026-07-01T14:59:33+09:00", "mjpeg": {"age_sec": 4.146, "exists": true, "mtime": "2026-07-01T14:59:33.970679+09:00", "mtime_epoch": 1782885573.9706786, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T14:59:38+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 860.055, "name": "celery", "pid": 267, "rss_mb": 856.207, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.658, "pushed": 0.658, "read": 0.658}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.703, "count": 14, "max_ms": 2.345, "p95_ms": 2.345}, "gstreamer_push": {"avg_ms": 1.203, "count": 14, "max_ms": 1.377, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.553, "count": 14, "max_ms": 52.799, "p95_ms": 52.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.739, "count": 14, "max_ms": 111.235, "p95_ms": 111.235}, "yolo11_640_fire_11class": {"avg_ms": 56.063, "count": 14, "max_ms": 82.801, "p95_ms": 82.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.783, "count": 14, "max_ms": 71.695, "p95_ms": 71.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.903, "count": 14, "max_ms": 121.522, "p95_ms": 121.522}, "read_interval": {"avg_ms": 2129.028, "count": 14, "max_ms": 15988.715, "p95_ms": 15988.715}, "triton_wait": {"avg_ms": 66.752, "count": 14, "max_ms": 111.249, "p95_ms": 111.249}, "visualize": {"avg_ms": 3.496, "count": 14, "max_ms": 4.761, "p95_ms": 4.761}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.276, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 24}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4048.777, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T14:58:15+09:00", "last_error_summary": "[h264 @ 0x55babb189ac0] error while decoding MB 75 39, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:58:15+09:00", "last_error_summary": "[h264 @ 0x55babb189ac0] error while decoding MB 75 39, bytestream -6", "last_exit_at": "2026-07-01T15:00:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.342, "last_started_at": "2026-07-01T14:58:18+09:00", "pid": 2243, "process": {"exists": false, "pid": 2243}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.708, "sample_count": 1227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2880, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:00:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T15:00:07+09:00", "last_frame_read_age_sec": 0.237, "last_frame_read_at": "2026-07-01T15:00:07+09:00", "mjpeg": {"age_sec": 0.225, "exists": true, "mtime": "2026-07-01T15:00:07.506825+09:00", "mtime_epoch": 1782885607.506825, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204720}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:00:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 858.934, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.325, "pushed": 4.325, "read": 4.291}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.186, "count": 129, "max_ms": 14.43, "p95_ms": 10.071}, "gstreamer_push": {"avg_ms": 0.936, "count": 129, "max_ms": 2.205, "p95_ms": 1.81}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.336, "count": 129, "max_ms": 77.007, "p95_ms": 67.685}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.419, "count": 129, "max_ms": 109.905, "p95_ms": 98.497}, "yolo11_640_fire_11class": {"avg_ms": 57.725, "count": 129, "max_ms": 86.975, "p95_ms": 80.072}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.828, "count": 129, "max_ms": 103.711, "p95_ms": 96.313}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.083, "count": 129, "max_ms": 300.584, "p95_ms": 266.708}, "read_interval": {"avg_ms": 232.795, "count": 128, "max_ms": 312.321, "p95_ms": 275.815}, "triton_wait": {"avg_ms": 75.957, "count": 129, "max_ms": 109.919, "p95_ms": 99.443}, "visualize": {"avg_ms": 9.941, "count": 129, "max_ms": 14.214, "p95_ms": 11.513}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.828, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 24}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4048.598, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:58:15+09:00", "last_error_summary": "[h264 @ 0x55babb189ac0] error while decoding MB 75 39, bytestream -6", "last_exit_at": "2026-07-01T15:00:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.346, "last_started_at": "2026-07-01T14:58:18+09:00", "pid": 2243, "process": {"exists": false, "pid": 2243}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.708, "sample_count": 1227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2880, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:00:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T15:00:07+09:00", "last_frame_read_age_sec": 0.24, "last_frame_read_at": "2026-07-01T15:00:07+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T15:00:07.506825+09:00", "mtime_epoch": 1782885607.506825, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204720}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:00:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 858.934, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.324, "pushed": 4.324, "read": 4.291}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.186, "count": 129, "max_ms": 14.43, "p95_ms": 10.071}, "gstreamer_push": {"avg_ms": 0.936, "count": 129, "max_ms": 2.205, "p95_ms": 1.81}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.336, "count": 129, "max_ms": 77.007, "p95_ms": 67.685}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.419, "count": 129, "max_ms": 109.905, "p95_ms": 98.497}, "yolo11_640_fire_11class": {"avg_ms": 57.725, "count": 129, "max_ms": 86.975, "p95_ms": 80.072}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.828, "count": 129, "max_ms": 103.711, "p95_ms": 96.313}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.083, "count": 129, "max_ms": 300.584, "p95_ms": 266.708}, "read_interval": {"avg_ms": 232.795, "count": 128, "max_ms": 312.321, "p95_ms": 275.815}, "triton_wait": {"avg_ms": 75.957, "count": 129, "max_ms": 109.919, "p95_ms": 99.443}, "visualize": {"avg_ms": 9.941, "count": 129, "max_ms": 14.214, "p95_ms": 11.513}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.832, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 24}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4159.328, "limit_mb": 8796093022207.996}, "detail": {"pid": 2502, "restart_count": 6, "start_count": 7}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T14:58:15+09:00", "last_error_summary": "[h264 @ 0x55babb189ac0] error while decoding MB 75 39, bytestream -6", "last_exit_at": "2026-07-01T15:00:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:00:10+09:00", "pid": 2502, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.164, "name": "ffmpeg", "pid": 2502, "rss_mb": 142.164, "state": "S (sleeping)", "threads": 106}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.708, "sample_count": 1208, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 2880, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:00:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.719, "last_frame_pushed_at": "2026-07-01T15:00:07+09:00", "last_frame_read_age_sec": 4.951, "last_frame_read_at": "2026-07-01T15:00:07+09:00", "mjpeg": {"age_sec": 4.699, "exists": true, "mtime": "2026-07-01T15:00:07.746826+09:00", "mtime_epoch": 1782885607.7468262, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204739}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:00:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 858.934, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.648, "pushed": 3.648, "read": 3.614}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.203, "count": 109, "max_ms": 14.43, "p95_ms": 10.107}, "gstreamer_push": {"avg_ms": 0.929, "count": 109, "max_ms": 2.205, "p95_ms": 1.771}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.459, "count": 109, "max_ms": 77.007, "p95_ms": 68.422}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.297, "count": 109, "max_ms": 109.905, "p95_ms": 98.81}, "yolo11_640_fire_11class": {"avg_ms": 58.057, "count": 109, "max_ms": 86.975, "p95_ms": 80.832}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.361, "count": 109, "max_ms": 103.711, "p95_ms": 96.503}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.973, "count": 109, "max_ms": 293.432, "p95_ms": 260.088}, "read_interval": {"avg_ms": 232.704, "count": 108, "max_ms": 305.715, "p95_ms": 270.503}, "triton_wait": {"avg_ms": 75.957, "count": 109, "max_ms": 109.919, "p95_ms": 99.547}, "visualize": {"avg_ms": 9.96, "count": 109, "max_ms": 14.214, "p95_ms": 11.909}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 27}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4050.141, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562dd9143c80] error while decoding MB 22 5, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:01:58+09:00", "last_error_summary": "[h264 @ 0x562dd9143c80] error while decoding MB 22 5, bytestream -5", "last_exit_at": "2026-07-01T15:00:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.105, "last_started_at": "2026-07-01T15:00:10+09:00", "pid": 2502, "process": {"exists": false, "pid": 2502}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.456, "sample_count": 1242, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:01:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T15:01:58+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T15:01:58+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T15:01:58.503475+09:00", "mtime_epoch": 1782885718.5034754, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203429}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:01:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.801, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.351, "pushed": 4.351, "read": 4.351}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.974, "count": 130, "max_ms": 14.702, "p95_ms": 9.406}, "gstreamer_push": {"avg_ms": 0.992, "count": 130, "max_ms": 3.017, "p95_ms": 1.87}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.351, "count": 130, "max_ms": 75.816, "p95_ms": 71.214}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.778, "count": 130, "max_ms": 119.936, "p95_ms": 103.022}, "yolo11_640_fire_11class": {"avg_ms": 59.835, "count": 130, "max_ms": 90.233, "p95_ms": 82.273}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.002, "count": 130, "max_ms": 107.42, "p95_ms": 101.114}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.538, "count": 130, "max_ms": 302.195, "p95_ms": 260.96}, "read_interval": {"avg_ms": 231.208, "count": 130, "max_ms": 314.098, "p95_ms": 270.225}, "triton_wait": {"avg_ms": 78.505, "count": 130, "max_ms": 119.948, "p95_ms": 104.405}, "visualize": {"avg_ms": 9.82, "count": 130, "max_ms": 12.71, "p95_ms": 11.151}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 27}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4048.977, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:01:58+09:00", "last_error_summary": "[h264 @ 0x562dd9143c80] error while decoding MB 22 5, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:01:58+09:00", "last_error_summary": "[h264 @ 0x562dd9143c80] error while decoding MB 22 5, bytestream -5", "last_exit_at": "2026-07-01T15:01:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.244, "last_started_at": "2026-07-01T15:00:10+09:00", "pid": 2502, "process": {"exists": false, "pid": 2502}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.456, "sample_count": 1243, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3338, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:01:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:01:58+09:00", "last_frame_read_age_sec": 0.191, "last_frame_read_at": "2026-07-01T15:01:58+09:00", "mjpeg": {"age_sec": 0.186, "exists": true, "mtime": "2026-07-01T15:01:58.503475+09:00", "mtime_epoch": 1782885718.5034754, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203429}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:01:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.801, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.373, "pushed": 4.373, "read": 4.34}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.969, "count": 130, "max_ms": 14.702, "p95_ms": 9.406}, "gstreamer_push": {"avg_ms": 0.992, "count": 130, "max_ms": 3.017, "p95_ms": 1.87}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.219, "count": 130, "max_ms": 75.816, "p95_ms": 71.214}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.534, "count": 130, "max_ms": 119.936, "p95_ms": 103.022}, "yolo11_640_fire_11class": {"avg_ms": 59.613, "count": 130, "max_ms": 90.233, "p95_ms": 82.273}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.777, "count": 130, "max_ms": 107.42, "p95_ms": 101.114}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.187, "count": 130, "max_ms": 302.195, "p95_ms": 260.96}, "read_interval": {"avg_ms": 231.132, "count": 129, "max_ms": 314.098, "p95_ms": 270.225}, "triton_wait": {"avg_ms": 78.266, "count": 130, "max_ms": 119.948, "p95_ms": 104.405}, "visualize": {"avg_ms": 9.831, "count": 130, "max_ms": 12.71, "p95_ms": 11.151}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.725, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 27}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4049.219, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:01:58+09:00", "last_error_summary": "[h264 @ 0x562dd9143c80] error while decoding MB 22 5, bytestream -5", "last_exit_at": "2026-07-01T15:01:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.248, "last_started_at": "2026-07-01T15:00:10+09:00", "pid": 2502, "process": {"exists": false, "pid": 2502}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.456, "sample_count": 1243, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3338, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:01:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:01:58+09:00", "last_frame_read_age_sec": 0.194, "last_frame_read_at": "2026-07-01T15:01:58+09:00", "mjpeg": {"age_sec": 0.189, "exists": true, "mtime": "2026-07-01T15:01:58.503475+09:00", "mtime_epoch": 1782885718.5034754, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203429}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:01:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.801, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.373, "pushed": 4.373, "read": 4.339}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.969, "count": 130, "max_ms": 14.702, "p95_ms": 9.406}, "gstreamer_push": {"avg_ms": 0.992, "count": 130, "max_ms": 3.017, "p95_ms": 1.87}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.219, "count": 130, "max_ms": 75.816, "p95_ms": 71.214}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.534, "count": 130, "max_ms": 119.936, "p95_ms": 103.022}, "yolo11_640_fire_11class": {"avg_ms": 59.613, "count": 130, "max_ms": 90.233, "p95_ms": 82.273}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.777, "count": 130, "max_ms": 107.42, "p95_ms": 101.114}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.187, "count": 130, "max_ms": 302.195, "p95_ms": 260.96}, "read_interval": {"avg_ms": 231.132, "count": 129, "max_ms": 314.098, "p95_ms": 270.225}, "triton_wait": {"avg_ms": 78.266, "count": 130, "max_ms": 119.948, "p95_ms": 104.405}, "visualize": {"avg_ms": 9.831, "count": 130, "max_ms": 12.71, "p95_ms": 11.151}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.728, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 27}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4156.625, "limit_mb": 8796093022207.996}, "detail": {"pid": 2609, "restart_count": 7, "start_count": 8}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:01:58+09:00", "last_error_summary": "[h264 @ 0x562dd9143c80] error while decoding MB 22 5, bytestream -5", "last_exit_at": "2026-07-01T15:02:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:02:00+09:00", "pid": 2609, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.23, "name": "ffmpeg", "pid": 2609, "rss_mb": 139.23, "state": "S (sleeping)", "threads": 106}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.456, "sample_count": 1225, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3338, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:01:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.065, "last_frame_pushed_at": "2026-07-01T15:01:58+09:00", "last_frame_read_age_sec": 4.253, "last_frame_read_at": "2026-07-01T15:01:58+09:00", "mjpeg": {"age_sec": 4.052, "exists": true, "mtime": "2026-07-01T15:01:58.699477+09:00", "mtime_epoch": 1782885718.6994767, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203497}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:02:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.801, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.784, "pushed": 3.784, "read": 3.751}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.85, "count": 113, "max_ms": 10.018, "p95_ms": 8.984}, "gstreamer_push": {"avg_ms": 0.999, "count": 113, "max_ms": 3.017, "p95_ms": 1.909}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.672, "count": 113, "max_ms": 75.816, "p95_ms": 71.214}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.712, "count": 113, "max_ms": 119.936, "p95_ms": 103.022}, "yolo11_640_fire_11class": {"avg_ms": 59.543, "count": 113, "max_ms": 90.233, "p95_ms": 82.485}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.735, "count": 113, "max_ms": 107.42, "p95_ms": 101.114}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.931, "count": 113, "max_ms": 302.195, "p95_ms": 260.96}, "read_interval": {"avg_ms": 230.892, "count": 112, "max_ms": 314.098, "p95_ms": 270.225}, "triton_wait": {"avg_ms": 78.43, "count": 113, "max_ms": 119.948, "p95_ms": 104.911}, "visualize": {"avg_ms": 9.819, "count": 113, "max_ms": 12.499, "p95_ms": 11.151}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.862, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 31}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4049.508, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5633e1b3ce00] error while decoding MB 15 12, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:03:51+09:00", "last_error_summary": "[h264 @ 0x5633e1b3ce00] error while decoding MB 15 12, bytestream -12", "last_exit_at": "2026-07-01T15:02:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.799, "last_started_at": "2026-07-01T15:02:00+09:00", "pid": 2609, "process": {"exists": false, "pid": 2609}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.456, "sample_count": 1247, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3796, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:03:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T15:03:51+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T15:03:51+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-01T15:03:51.512359+09:00", "mtime_epoch": 1782885831.512359, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208492}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:03:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.121, "pushed": 4.121, "read": 4.121}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.79, "count": 123, "max_ms": 12.388, "p95_ms": 10.684}, "gstreamer_push": {"avg_ms": 0.931, "count": 123, "max_ms": 2.957, "p95_ms": 1.186}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.589, "count": 123, "max_ms": 75.634, "p95_ms": 66.167}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.425, "count": 123, "max_ms": 113.58, "p95_ms": 101.71}, "yolo11_640_fire_11class": {"avg_ms": 57.846, "count": 123, "max_ms": 84.538, "p95_ms": 78.782}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.878, "count": 123, "max_ms": 100.858, "p95_ms": 95.817}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.534, "count": 123, "max_ms": 330.486, "p95_ms": 273.324}, "read_interval": {"avg_ms": 244.282, "count": 123, "max_ms": 339.583, "p95_ms": 282.495}, "triton_wait": {"avg_ms": 75.864, "count": 123, "max_ms": 113.593, "p95_ms": 101.723}, "visualize": {"avg_ms": 10.765, "count": 123, "max_ms": 15.876, "p95_ms": 12.636}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.844, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 31}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4051.016, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:03:51+09:00", "last_error_summary": "[h264 @ 0x5633e1b3ce00] error while decoding MB 15 12, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:03:51+09:00", "last_error_summary": "[h264 @ 0x5633e1b3ce00] error while decoding MB 15 12, bytestream -12", "last_exit_at": "2026-07-01T15:03:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.985, "last_started_at": "2026-07-01T15:02:00+09:00", "pid": 2609, "process": {"exists": false, "pid": 2609}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.456, "sample_count": 1247, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3797, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:03:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T15:03:51+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T15:03:51+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T15:03:51.512359+09:00", "mtime_epoch": 1782885831.512359, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208492}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:03:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.293, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.127, "pushed": 4.127, "read": 4.093}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.804, "count": 123, "max_ms": 12.388, "p95_ms": 10.684}, "gstreamer_push": {"avg_ms": 0.931, "count": 123, "max_ms": 2.957, "p95_ms": 1.186}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.541, "count": 123, "max_ms": 75.634, "p95_ms": 66.167}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.16, "count": 123, "max_ms": 113.58, "p95_ms": 101.71}, "yolo11_640_fire_11class": {"avg_ms": 57.646, "count": 123, "max_ms": 84.538, "p95_ms": 77.692}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.618, "count": 123, "max_ms": 100.858, "p95_ms": 90.703}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.348, "count": 123, "max_ms": 330.486, "p95_ms": 273.324}, "read_interval": {"avg_ms": 244.133, "count": 122, "max_ms": 339.583, "p95_ms": 282.495}, "triton_wait": {"avg_ms": 75.6, "count": 123, "max_ms": 113.593, "p95_ms": 101.723}, "visualize": {"avg_ms": 10.776, "count": 123, "max_ms": 15.876, "p95_ms": 12.636}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.807, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 31}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4051.129, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:03:51+09:00", "last_error_summary": "[h264 @ 0x5633e1b3ce00] error while decoding MB 15 12, bytestream -12", "last_exit_at": "2026-07-01T15:03:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.988, "last_started_at": "2026-07-01T15:02:00+09:00", "pid": 2609, "process": {"exists": false, "pid": 2609}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.456, "sample_count": 1247, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3797, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:03:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T15:03:51+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T15:03:51+09:00", "mjpeg": {"age_sec": 0.227, "exists": true, "mtime": "2026-07-01T15:03:51.512359+09:00", "mtime_epoch": 1782885831.512359, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208492}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:03:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.293, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.126, "pushed": 4.126, "read": 4.093}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.804, "count": 123, "max_ms": 12.388, "p95_ms": 10.684}, "gstreamer_push": {"avg_ms": 0.931, "count": 123, "max_ms": 2.957, "p95_ms": 1.186}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.541, "count": 123, "max_ms": 75.634, "p95_ms": 66.167}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.16, "count": 123, "max_ms": 113.58, "p95_ms": 101.71}, "yolo11_640_fire_11class": {"avg_ms": 57.646, "count": 123, "max_ms": 84.538, "p95_ms": 77.692}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.618, "count": 123, "max_ms": 100.858, "p95_ms": 90.703}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.348, "count": 123, "max_ms": 330.486, "p95_ms": 273.324}, "read_interval": {"avg_ms": 244.133, "count": 122, "max_ms": 339.583, "p95_ms": 282.495}, "triton_wait": {"avg_ms": 75.6, "count": 123, "max_ms": 113.593, "p95_ms": 101.723}, "visualize": {"avg_ms": 10.776, "count": 123, "max_ms": 15.876, "p95_ms": 12.636}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.81, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 31}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4157.262, "limit_mb": 8796093022207.996}, "detail": {"pid": 2716, "restart_count": 8, "start_count": 9}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:03:51+09:00", "last_error_summary": "[h264 @ 0x5633e1b3ce00] error while decoding MB 15 12, bytestream -12", "last_exit_at": "2026-07-01T15:03:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:03:53+09:00", "pid": 2716, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.738, "name": "ffmpeg", "pid": 2716, "rss_mb": 136.738, "state": "S (sleeping)", "threads": 106}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.153, "sample_count": 1231, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 3797, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:03:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.023, "last_frame_pushed_at": "2026-07-01T15:03:51+09:00", "last_frame_read_age_sec": 4.255, "last_frame_read_at": "2026-07-01T15:03:51+09:00", "mjpeg": {"age_sec": 4.008, "exists": true, "mtime": "2026-07-01T15:03:51.748361+09:00", "mtime_epoch": 1782885831.7483609, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208431}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:03:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.554, "pushed": 3.554, "read": 3.52}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.825, "count": 106, "max_ms": 12.388, "p95_ms": 10.684}, "gstreamer_push": {"avg_ms": 0.931, "count": 106, "max_ms": 2.957, "p95_ms": 1.186}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.849, "count": 106, "max_ms": 75.634, "p95_ms": 66.651}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.528, "count": 106, "max_ms": 113.58, "p95_ms": 102.206}, "yolo11_640_fire_11class": {"avg_ms": 58.01, "count": 106, "max_ms": 84.538, "p95_ms": 80.054}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.816, "count": 106, "max_ms": 100.858, "p95_ms": 97.509}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.838, "count": 106, "max_ms": 330.486, "p95_ms": 275.344}, "read_interval": {"avg_ms": 245.657, "count": 105, "max_ms": 339.583, "p95_ms": 286.228}, "triton_wait": {"avg_ms": 75.993, "count": 106, "max_ms": 113.593, "p95_ms": 102.218}, "visualize": {"avg_ms": 10.858, "count": 106, "max_ms": 15.876, "p95_ms": 12.678}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.829, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4053.594, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5611fe09c780] error while decoding MB 8 10, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:05:41+09:00", "last_error_summary": "[h264 @ 0x5611fe09c780] error while decoding MB 8 10, bytestream -9", "last_exit_at": "2026-07-01T15:03:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.109, "last_started_at": "2026-07-01T15:03:53+09:00", "pid": 2716, "process": {"exists": false, "pid": 2716}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 262.6, "sample_count": 1250, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4256, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:05:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.053, "last_frame_pushed_at": "2026-07-01T15:05:41+09:00", "last_frame_read_age_sec": 0.045, "last_frame_read_at": "2026-07-01T15:05:41+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T15:05:41.821395+09:00", "mtime_epoch": 1782885941.8213947, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199224}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:05:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.562, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.316, "pushed": 4.316, "read": 4.316}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.977, "count": 128, "max_ms": 11.714, "p95_ms": 9.39}, "gstreamer_push": {"avg_ms": 0.945, "count": 129, "max_ms": 2.916, "p95_ms": 1.831}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.978, "count": 128, "max_ms": 78.931, "p95_ms": 73.279}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.494, "count": 128, "max_ms": 110.097, "p95_ms": 102.81}, "yolo11_640_fire_11class": {"avg_ms": 57.111, "count": 128, "max_ms": 83.379, "p95_ms": 77.945}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.614, "count": 128, "max_ms": 102.523, "p95_ms": 99.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.214, "count": 129, "max_ms": 322.081, "p95_ms": 255.948}, "read_interval": {"avg_ms": 232.976, "count": 129, "max_ms": 338.575, "p95_ms": 266.625}, "triton_wait": {"avg_ms": 76.255, "count": 128, "max_ms": 110.111, "p95_ms": 102.825}, "visualize": {"avg_ms": 9.942, "count": 128, "max_ms": 12.395, "p95_ms": 11.24}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4052.477, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:05:41+09:00", "last_error_summary": "[h264 @ 0x5611fe09c780] error while decoding MB 8 10, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:05:41+09:00", "last_error_summary": "[h264 @ 0x5611fe09c780] error while decoding MB 8 10, bytestream -9", "last_exit_at": "2026-07-01T15:05:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.271, "last_started_at": "2026-07-01T15:03:53+09:00", "pid": 2716, "process": {"exists": false, "pid": 2716}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 262.6, "sample_count": 1251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4257, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:05:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:05:42+09:00", "last_frame_read_age_sec": 0.206, "last_frame_read_at": "2026-07-01T15:05:41+09:00", "mjpeg": {"age_sec": 0.204, "exists": true, "mtime": "2026-07-01T15:05:41.821395+09:00", "mtime_epoch": 1782885941.8213947, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199224}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:05:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.562, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.33, "pushed": 4.33, "read": 4.296}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.97, "count": 129, "max_ms": 11.714, "p95_ms": 9.39}, "gstreamer_push": {"avg_ms": 0.945, "count": 129, "max_ms": 2.916, "p95_ms": 1.831}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.072, "count": 129, "max_ms": 78.931, "p95_ms": 73.279}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.507, "count": 129, "max_ms": 110.097, "p95_ms": 102.81}, "yolo11_640_fire_11class": {"avg_ms": 57.132, "count": 129, "max_ms": 83.379, "p95_ms": 77.945}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.646, "count": 129, "max_ms": 102.523, "p95_ms": 99.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.162, "count": 129, "max_ms": 322.081, "p95_ms": 255.948}, "read_interval": {"avg_ms": 233.096, "count": 128, "max_ms": 338.575, "p95_ms": 266.625}, "triton_wait": {"avg_ms": 76.262, "count": 129, "max_ms": 110.111, "p95_ms": 102.825}, "visualize": {"avg_ms": 9.934, "count": 129, "max_ms": 12.395, "p95_ms": 11.24}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.796, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4052.402, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:05:41+09:00", "last_error_summary": "[h264 @ 0x5611fe09c780] error while decoding MB 8 10, bytestream -9", "last_exit_at": "2026-07-01T15:05:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.274, "last_started_at": "2026-07-01T15:03:53+09:00", "pid": 2716, "process": {"exists": false, "pid": 2716}, "restart_count": 8, "restart_count_per_hour": 8, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 262.6, "sample_count": 1251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4257, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:05:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:05:42+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T15:05:41+09:00", "mjpeg": {"age_sec": 0.207, "exists": true, "mtime": "2026-07-01T15:05:41.821395+09:00", "mtime_epoch": 1782885941.8213947, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199224}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:05:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.562, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.329, "pushed": 4.329, "read": 4.295}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.97, "count": 129, "max_ms": 11.714, "p95_ms": 9.39}, "gstreamer_push": {"avg_ms": 0.945, "count": 129, "max_ms": 2.916, "p95_ms": 1.831}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.072, "count": 129, "max_ms": 78.931, "p95_ms": 73.279}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.507, "count": 129, "max_ms": 110.097, "p95_ms": 102.81}, "yolo11_640_fire_11class": {"avg_ms": 57.132, "count": 129, "max_ms": 83.379, "p95_ms": 77.945}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.646, "count": 129, "max_ms": 102.523, "p95_ms": 99.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.162, "count": 129, "max_ms": 322.081, "p95_ms": 255.948}, "read_interval": {"avg_ms": 233.096, "count": 128, "max_ms": 338.575, "p95_ms": 266.625}, "triton_wait": {"avg_ms": 76.262, "count": 129, "max_ms": 110.111, "p95_ms": 102.825}, "visualize": {"avg_ms": 9.934, "count": 129, "max_ms": 12.395, "p95_ms": 11.24}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.799, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 35}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4159.805, "limit_mb": 8796093022207.996}, "detail": {"pid": 2823, "restart_count": 9, "start_count": 10}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:05:41+09:00", "last_error_summary": "[h264 @ 0x5611fe09c780] error while decoding MB 8 10, bytestream -9", "last_exit_at": "2026-07-01T15:05:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:05:44+09:00", "pid": 2823, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.34, "name": "ffmpeg", "pid": 2823, "rss_mb": 139.34, "state": "S (sleeping)", "threads": 106}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 262.6, "sample_count": 1232, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4257, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:05:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.419, "last_frame_pushed_at": "2026-07-01T15:05:42+09:00", "last_frame_read_age_sec": 4.622, "last_frame_read_at": "2026-07-01T15:05:41+09:00", "mjpeg": {"age_sec": 4.408, "exists": true, "mtime": "2026-07-01T15:05:42.033397+09:00", "mtime_epoch": 1782885942.033397, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190964}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:05:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 856.562, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.689, "pushed": 3.689, "read": 3.656}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.01, "count": 110, "max_ms": 11.714, "p95_ms": 9.778}, "gstreamer_push": {"avg_ms": 0.955, "count": 110, "max_ms": 2.916, "p95_ms": 1.862}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.613, "count": 110, "max_ms": 78.931, "p95_ms": 71.738}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.214, "count": 110, "max_ms": 110.097, "p95_ms": 102.252}, "yolo11_640_fire_11class": {"avg_ms": 57.122, "count": 110, "max_ms": 83.379, "p95_ms": 77.945}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.85, "count": 110, "max_ms": 102.523, "p95_ms": 99.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.015, "count": 110, "max_ms": 322.081, "p95_ms": 255.948}, "read_interval": {"avg_ms": 233.005, "count": 109, "max_ms": 338.575, "p95_ms": 266.625}, "triton_wait": {"avg_ms": 76.037, "count": 110, "max_ms": 110.111, "p95_ms": 102.265}, "visualize": {"avg_ms": 9.913, "count": 110, "max_ms": 12.395, "p95_ms": 11.554}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.816, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 39}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4052.77, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5604301c65c0] error while decoding MB 54 4, bytestream -16", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:07:33+09:00", "last_error_summary": "[h264 @ 0x5604301c65c0] error while decoding MB 54 4, bytestream -16", "last_exit_at": "2026-07-01T15:05:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.9, "last_started_at": "2026-07-01T15:05:44+09:00", "pid": 2823, "process": {"exists": false, "pid": 2823}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 260.14, "sample_count": 1251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4714, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:07:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.054, "last_frame_pushed_at": "2026-07-01T15:07:33+09:00", "last_frame_read_age_sec": 0.046, "last_frame_read_at": "2026-07-01T15:07:33+09:00", "mjpeg": {"age_sec": 0.03, "exists": true, "mtime": "2026-07-01T15:07:33.310582+09:00", "mtime_epoch": 1782886053.3105817, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213425}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:07:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 859.359, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.32, "pushed": 4.32, "read": 4.32}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.018, "count": 129, "max_ms": 10.501, "p95_ms": 9.753}, "gstreamer_push": {"avg_ms": 0.958, "count": 129, "max_ms": 3.405, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.876, "count": 129, "max_ms": 79.546, "p95_ms": 71.676}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.015, "count": 129, "max_ms": 114.864, "p95_ms": 100.872}, "yolo11_640_fire_11class": {"avg_ms": 60.017, "count": 129, "max_ms": 89.451, "p95_ms": 82.77}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.529, "count": 129, "max_ms": 109.965, "p95_ms": 96.84}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.826, "count": 129, "max_ms": 305.92, "p95_ms": 251.606}, "read_interval": {"avg_ms": 232.728, "count": 129, "max_ms": 321.1, "p95_ms": 261.871}, "triton_wait": {"avg_ms": 76.747, "count": 129, "max_ms": 114.877, "p95_ms": 102.608}, "visualize": {"avg_ms": 10.048, "count": 129, "max_ms": 12.799, "p95_ms": 12.254}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.862, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 39}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4051.664, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:07:33+09:00", "last_error_summary": "[h264 @ 0x5604301c65c0] error while decoding MB 54 4, bytestream -16", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:07:33+09:00", "last_error_summary": "[h264 @ 0x5604301c65c0] error while decoding MB 54 4, bytestream -16", "last_exit_at": "2026-07-01T15:07:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.074, "last_started_at": "2026-07-01T15:05:44+09:00", "pid": 2823, "process": {"exists": false, "pid": 2823}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 260.14, "sample_count": 1251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4715, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:07:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:07:33+09:00", "last_frame_read_age_sec": 0.221, "last_frame_read_at": "2026-07-01T15:07:33+09:00", "mjpeg": {"age_sec": 0.204, "exists": true, "mtime": "2026-07-01T15:07:33.310582+09:00", "mtime_epoch": 1782886053.3105817, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213425}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:07:33+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 859.359, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.332, "pushed": 4.332, "read": 4.298}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.022, "count": 129, "max_ms": 10.501, "p95_ms": 9.753}, "gstreamer_push": {"avg_ms": 0.958, "count": 129, "max_ms": 3.405, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.857, "count": 129, "max_ms": 79.546, "p95_ms": 71.676}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.95, "count": 129, "max_ms": 114.864, "p95_ms": 100.872}, "yolo11_640_fire_11class": {"avg_ms": 60.028, "count": 129, "max_ms": 89.451, "p95_ms": 82.77}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.587, "count": 129, "max_ms": 109.965, "p95_ms": 96.84}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.899, "count": 129, "max_ms": 305.92, "p95_ms": 251.606}, "read_interval": {"avg_ms": 232.861, "count": 128, "max_ms": 321.1, "p95_ms": 261.871}, "triton_wait": {"avg_ms": 76.8, "count": 129, "max_ms": 114.877, "p95_ms": 102.608}, "visualize": {"avg_ms": 10.044, "count": 129, "max_ms": 12.799, "p95_ms": 12.254}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.779, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 39}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4051.656, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:07:33+09:00", "last_error_summary": "[h264 @ 0x5604301c65c0] error while decoding MB 54 4, bytestream -16", "last_exit_at": "2026-07-01T15:07:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.078, "last_started_at": "2026-07-01T15:05:44+09:00", "pid": 2823, "process": {"exists": false, "pid": 2823}, "restart_count": 9, "restart_count_per_hour": 9, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 260.14, "sample_count": 1251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4715, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:07:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T15:07:33+09:00", "last_frame_read_age_sec": 0.224, "last_frame_read_at": "2026-07-01T15:07:33+09:00", "mjpeg": {"age_sec": 0.207, "exists": true, "mtime": "2026-07-01T15:07:33.310582+09:00", "mtime_epoch": 1782886053.3105817, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213425}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:07:33+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 859.359, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.331, "pushed": 4.331, "read": 4.298}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.022, "count": 129, "max_ms": 10.501, "p95_ms": 9.753}, "gstreamer_push": {"avg_ms": 0.958, "count": 129, "max_ms": 3.405, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.857, "count": 129, "max_ms": 79.546, "p95_ms": 71.676}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.95, "count": 129, "max_ms": 114.864, "p95_ms": 100.872}, "yolo11_640_fire_11class": {"avg_ms": 60.028, "count": 129, "max_ms": 89.451, "p95_ms": 82.77}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.587, "count": 129, "max_ms": 109.965, "p95_ms": 96.84}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.899, "count": 129, "max_ms": 305.92, "p95_ms": 251.606}, "read_interval": {"avg_ms": 232.861, "count": 128, "max_ms": 321.1, "p95_ms": 261.871}, "triton_wait": {"avg_ms": 76.8, "count": 129, "max_ms": 114.877, "p95_ms": 102.608}, "visualize": {"avg_ms": 10.044, "count": 129, "max_ms": 12.799, "p95_ms": 12.254}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.782, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 39}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4158.602, "limit_mb": 8796093022207.996}, "detail": {"pid": 2930, "restart_count": 10, "start_count": 11}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:07:33+09:00", "last_error_summary": "[h264 @ 0x5604301c65c0] error while decoding MB 54 4, bytestream -16", "last_exit_at": "2026-07-01T15:07:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:07:36+09:00", "pid": 2930, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.555, "name": "ffmpeg", "pid": 2930, "rss_mb": 136.555, "state": "S (sleeping)", "threads": 106}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 260.14, "sample_count": 1230, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 4715, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:07:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.864, "last_frame_pushed_at": "2026-07-01T15:07:33+09:00", "last_frame_read_age_sec": 5.082, "last_frame_read_at": "2026-07-01T15:07:33+09:00", "mjpeg": {"age_sec": 4.845, "exists": true, "mtime": "2026-07-01T15:07:33.530584+09:00", "mtime_epoch": 1782886053.530584, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213290}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:07:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 861.402, "name": "celery", "pid": 237, "rss_mb": 859.359, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.623, "pushed": 3.623, "read": 3.59}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.033, "count": 108, "max_ms": 10.501, "p95_ms": 9.811}, "gstreamer_push": {"avg_ms": 0.949, "count": 108, "max_ms": 3.405, "p95_ms": 1.311}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.029, "count": 108, "max_ms": 79.546, "p95_ms": 72.113}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.201, "count": 108, "max_ms": 114.864, "p95_ms": 100.872}, "yolo11_640_fire_11class": {"avg_ms": 60.166, "count": 108, "max_ms": 87.288, "p95_ms": 82.77}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.064, "count": 108, "max_ms": 109.965, "p95_ms": 97.843}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.122, "count": 108, "max_ms": 305.92, "p95_ms": 251.606}, "read_interval": {"avg_ms": 233.105, "count": 107, "max_ms": 321.1, "p95_ms": 261.871}, "triton_wait": {"avg_ms": 77.134, "count": 108, "max_ms": 114.877, "p95_ms": 102.608}, "visualize": {"avg_ms": 10.059, "count": 108, "max_ms": 12.799, "p95_ms": 12.255}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.807, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 43}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4304.281, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55df7baf5900] error while decoding MB 79 22, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:09:24+09:00", "last_error_summary": "[h264 @ 0x55df7baf5900] error while decoding MB 79 22, bytestream -5", "last_exit_at": "2026-07-01T15:07:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.366, "last_started_at": "2026-07-01T15:07:36+09:00", "pid": 2930, "process": {"exists": false, "pid": 2930}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 257.18, "sample_count": 1252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5173, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:09:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T15:09:24+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T15:09:24+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T15:09:24.699880+09:00", "mtime_epoch": 1782886164.6998804, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193573}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:09:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 862.75, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.394, "pushed": 4.394, "read": 4.394}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.085, "count": 131, "max_ms": 11.122, "p95_ms": 10.027}, "gstreamer_push": {"avg_ms": 0.921, "count": 131, "max_ms": 2.802, "p95_ms": 1.16}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.555, "count": 131, "max_ms": 93.333, "p95_ms": 69.612}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.418, "count": 131, "max_ms": 110.553, "p95_ms": 102.642}, "yolo11_640_fire_11class": {"avg_ms": 55.749, "count": 131, "max_ms": 83.995, "p95_ms": 74.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.203, "count": 131, "max_ms": 115.908, "p95_ms": 93.012}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.167, "count": 131, "max_ms": 318.44, "p95_ms": 252.429}, "read_interval": {"avg_ms": 228.965, "count": 131, "max_ms": 328.701, "p95_ms": 261.636}, "triton_wait": {"avg_ms": 74.015, "count": 131, "max_ms": 115.921, "p95_ms": 103.582}, "visualize": {"avg_ms": 9.974, "count": 131, "max_ms": 13.72, "p95_ms": 12.126}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.816, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 43}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4303.531, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:09:24+09:00", "last_error_summary": "[h264 @ 0x55df7baf5900] error while decoding MB 79 22, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:09:24+09:00", "last_error_summary": "[h264 @ 0x55df7baf5900] error while decoding MB 79 22, bytestream -5", "last_exit_at": "2026-07-01T15:09:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.56, "last_started_at": "2026-07-01T15:07:36+09:00", "pid": 2930, "process": {"exists": false, "pid": 2930}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 257.18, "sample_count": 1253, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5174, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:09:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:09:24+09:00", "last_frame_read_age_sec": 0.247, "last_frame_read_at": "2026-07-01T15:09:24+09:00", "mjpeg": {"age_sec": 0.235, "exists": true, "mtime": "2026-07-01T15:09:24.699880+09:00", "mtime_epoch": 1782886164.6998804, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193573}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:09:24+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 862.75, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.397, "pushed": 4.397, "read": 4.363}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.086, "count": 131, "max_ms": 11.122, "p95_ms": 10.027}, "gstreamer_push": {"avg_ms": 0.921, "count": 131, "max_ms": 2.802, "p95_ms": 1.16}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.624, "count": 131, "max_ms": 93.333, "p95_ms": 69.612}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.496, "count": 131, "max_ms": 110.553, "p95_ms": 102.642}, "yolo11_640_fire_11class": {"avg_ms": 55.81, "count": 131, "max_ms": 83.995, "p95_ms": 74.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.21, "count": 131, "max_ms": 115.908, "p95_ms": 93.012}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.253, "count": 131, "max_ms": 318.44, "p95_ms": 252.429}, "read_interval": {"avg_ms": 228.878, "count": 130, "max_ms": 328.701, "p95_ms": 261.636}, "triton_wait": {"avg_ms": 74.094, "count": 131, "max_ms": 115.921, "p95_ms": 103.582}, "visualize": {"avg_ms": 9.97, "count": 131, "max_ms": 13.72, "p95_ms": 12.126}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.796, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 43}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4303.414, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:09:24+09:00", "last_error_summary": "[h264 @ 0x55df7baf5900] error while decoding MB 79 22, bytestream -5", "last_exit_at": "2026-07-01T15:09:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.564, "last_started_at": "2026-07-01T15:07:36+09:00", "pid": 2930, "process": {"exists": false, "pid": 2930}, "restart_count": 10, "restart_count_per_hour": 10, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 257.18, "sample_count": 1253, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5174, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:09:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:09:24+09:00", "last_frame_read_age_sec": 0.25, "last_frame_read_at": "2026-07-01T15:09:24+09:00", "mjpeg": {"age_sec": 0.239, "exists": true, "mtime": "2026-07-01T15:09:24.699880+09:00", "mtime_epoch": 1782886164.6998804, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193573}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:09:24+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 862.75, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.396, "pushed": 4.396, "read": 4.363}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.086, "count": 131, "max_ms": 11.122, "p95_ms": 10.027}, "gstreamer_push": {"avg_ms": 0.921, "count": 131, "max_ms": 2.802, "p95_ms": 1.16}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.624, "count": 131, "max_ms": 93.333, "p95_ms": 69.612}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.496, "count": 131, "max_ms": 110.553, "p95_ms": 102.642}, "yolo11_640_fire_11class": {"avg_ms": 55.81, "count": 131, "max_ms": 83.995, "p95_ms": 74.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.21, "count": 131, "max_ms": 115.908, "p95_ms": 93.012}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.253, "count": 131, "max_ms": 318.44, "p95_ms": 252.429}, "read_interval": {"avg_ms": 228.878, "count": 130, "max_ms": 328.701, "p95_ms": 261.636}, "triton_wait": {"avg_ms": 74.094, "count": 131, "max_ms": 115.921, "p95_ms": 103.582}, "visualize": {"avg_ms": 9.97, "count": 131, "max_ms": 13.72, "p95_ms": 12.126}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.799, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 43}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4411.383, "limit_mb": 8796093022207.996}, "detail": {"pid": 3098, "restart_count": 11, "start_count": 12}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:09:24+09:00", "last_error_summary": "[h264 @ 0x55df7baf5900] error while decoding MB 79 22, bytestream -5", "last_exit_at": "2026-07-01T15:09:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:09:27+09:00", "pid": 3098, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.332, "name": "ffmpeg", "pid": 3098, "rss_mb": 139.332, "state": "S (sleeping)", "threads": 106}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 257.18, "sample_count": 1235, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5174, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:09:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.089, "last_frame_pushed_at": "2026-07-01T15:09:24+09:00", "last_frame_read_age_sec": 4.333, "last_frame_read_at": "2026-07-01T15:09:24+09:00", "mjpeg": {"age_sec": 4.07, "exists": true, "mtime": "2026-07-01T15:09:24.951884+09:00", "mtime_epoch": 1782886164.9518836, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193510}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:09:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 862.75, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.788, "pushed": 3.788, "read": 3.755}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.183, "count": 113, "max_ms": 11.122, "p95_ms": 10.17}, "gstreamer_push": {"avg_ms": 0.91, "count": 113, "max_ms": 1.99, "p95_ms": 1.16}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.575, "count": 113, "max_ms": 93.333, "p95_ms": 69.888}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.754, "count": 113, "max_ms": 110.553, "p95_ms": 102.845}, "yolo11_640_fire_11class": {"avg_ms": 56.621, "count": 113, "max_ms": 83.995, "p95_ms": 76.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.299, "count": 113, "max_ms": 115.908, "p95_ms": 93.468}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.362, "count": 113, "max_ms": 318.44, "p95_ms": 252.429}, "read_interval": {"avg_ms": 229.946, "count": 112, "max_ms": 328.701, "p95_ms": 261.636}, "triton_wait": {"avg_ms": 75.415, "count": 113, "max_ms": 115.921, "p95_ms": 105.505}, "visualize": {"avg_ms": 9.967, "count": 113, "max_ms": 13.72, "p95_ms": 12.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.83, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4237.137, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55777acf5880] error while decoding MB 35 39, bytestream -25 [in#0/rtsp @ 0x55777a4b3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55777a4b3e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:11:03+09:00", "last_error_summary": "[h264 @ 0x55777acf5880] error while decoding MB 35 39, bytestream -25 [in#0/rtsp @ 0x55777a4b3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55777a4b3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T14:59:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 687.561, "last_started_at": "2026-07-01T14:59:36+09:00", "pid": 2350, "process": {"exists": false, "pid": 2350}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.317, "sample_count": 2565, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12211, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:11:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-01T15:11:03+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-01T15:11:03+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-01T15:11:03.621111+09:00", "mtime_epoch": 1782886263.6211107, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257224}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T15:11:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1129.996, "name": "celery", "pid": 267, "rss_mb": 1092.898, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.688, "pushed": 0.688, "read": 0.745}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.741, "count": 12, "max_ms": 2.266, "p95_ms": 2.266}, "gstreamer_push": {"avg_ms": 1.341, "count": 12, "max_ms": 1.583, "p95_ms": 1.583}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.348, "count": 12, "max_ms": 53.417, "p95_ms": 53.417}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.136, "count": 12, "max_ms": 95.406, "p95_ms": 95.406}, "yolo11_640_fire_11class": {"avg_ms": 58.812, "count": 12, "max_ms": 86.328, "p95_ms": 86.328}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.447, "count": 12, "max_ms": 104.163, "p95_ms": 104.163}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.973, "count": 12, "max_ms": 114.047, "p95_ms": 114.047}, "read_interval": {"avg_ms": 2737.977, "count": 13, "max_ms": 18203.694, "p95_ms": 18203.694}, "triton_wait": {"avg_ms": 69.532, "count": 12, "max_ms": 104.175, "p95_ms": 104.175}, "visualize": {"avg_ms": 3.708, "count": 12, "max_ms": 4.349, "p95_ms": 4.349}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.449, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4236.766, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:11:03+09:00", "last_error_summary": "[h264 @ 0x55777acf5880] error while decoding MB 35 39, bytestream -25 [in#0/rtsp @ 0x55777a4b3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55777a4b3e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:11:03+09:00", "last_error_summary": "[h264 @ 0x55777acf5880] error while decoding MB 35 39, bytestream -25 [in#0/rtsp @ 0x55777a4b3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55777a4b3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:11:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 687.584, "last_started_at": "2026-07-01T14:59:36+09:00", "pid": 2350, "process": {"exists": false, "pid": 2350}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.317, "sample_count": 2565, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12212, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:11:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T15:11:03+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-01T15:11:03+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-01T15:11:03.621111+09:00", "mtime_epoch": 1782886263.6211107, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257224}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T15:11:03+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1129.996, "name": "celery", "pid": 267, "rss_mb": 1092.898, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.794, "count": 13, "max_ms": 2.431, "p95_ms": 2.431}, "gstreamer_push": {"avg_ms": 1.322, "count": 13, "max_ms": 1.583, "p95_ms": 1.583}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.139, "count": 13, "max_ms": 53.417, "p95_ms": 53.417}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.146, "count": 13, "max_ms": 95.406, "p95_ms": 95.406}, "yolo11_640_fire_11class": {"avg_ms": 58.412, "count": 13, "max_ms": 86.328, "p95_ms": 86.328}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.027, "count": 13, "max_ms": 104.163, "p95_ms": 104.163}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.207, "count": 13, "max_ms": 114.047, "p95_ms": 114.047}, "read_interval": {"avg_ms": 2737.977, "count": 13, "max_ms": 18203.694, "p95_ms": 18203.694}, "triton_wait": {"avg_ms": 68.436, "count": 13, "max_ms": 104.175, "p95_ms": 104.175}, "visualize": {"avg_ms": 3.924, "count": 13, "max_ms": 6.521, "p95_ms": 6.521}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.472, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4236.508, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:11:03+09:00", "last_error_summary": "[h264 @ 0x55777acf5880] error while decoding MB 35 39, bytestream -25 [in#0/rtsp @ 0x55777a4b3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55777a4b3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:11:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 687.592, "last_started_at": "2026-07-01T14:59:36+09:00", "pid": 2350, "process": {"exists": false, "pid": 2350}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.317, "sample_count": 2565, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12212, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:11:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-01T15:11:03+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-01T15:11:03+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-01T15:11:03.621111+09:00", "mtime_epoch": 1782886263.6211107, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257224}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T15:11:03+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1129.996, "name": "celery", "pid": 267, "rss_mb": 1092.898, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.794, "count": 13, "max_ms": 2.431, "p95_ms": 2.431}, "gstreamer_push": {"avg_ms": 1.322, "count": 13, "max_ms": 1.583, "p95_ms": 1.583}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.139, "count": 13, "max_ms": 53.417, "p95_ms": 53.417}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.146, "count": 13, "max_ms": 95.406, "p95_ms": 95.406}, "yolo11_640_fire_11class": {"avg_ms": 58.412, "count": 13, "max_ms": 86.328, "p95_ms": 86.328}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.027, "count": 13, "max_ms": 104.163, "p95_ms": 104.163}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.207, "count": 13, "max_ms": 114.047, "p95_ms": 114.047}, "read_interval": {"avg_ms": 2737.977, "count": 13, "max_ms": 18203.694, "p95_ms": 18203.694}, "triton_wait": {"avg_ms": 68.436, "count": 13, "max_ms": 104.175, "p95_ms": 104.175}, "visualize": {"avg_ms": 3.924, "count": 13, "max_ms": 6.521, "p95_ms": 6.521}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.48, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4258.359, "limit_mb": 8796093022207.996}, "detail": {"pid": 3205, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:11:03+09:00", "last_error_summary": "[h264 @ 0x55777acf5880] error while decoding MB 35 39, bytestream -25 [in#0/rtsp @ 0x55777a4b3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55777a4b3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:11:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:11:06+09:00", "pid": 3205, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.297, "name": "ffmpeg", "pid": 3205, "rss_mb": 58.297, "state": "S (sleeping)", "threads": 1}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 156.776, "sample_count": 2520, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12212, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:11:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.501, "last_frame_pushed_at": "2026-07-01T15:11:03+09:00", "last_frame_read_age_sec": 4.572, "last_frame_read_at": "2026-07-01T15:11:03+09:00", "mjpeg": {"age_sec": 4.484, "exists": true, "mtime": "2026-07-01T15:11:03.705112+09:00", "mtime_epoch": 1782886263.7051117, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257158}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T15:11:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1129.996, "name": "celery", "pid": 267, "rss_mb": 1092.898, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.592, "pushed": 0.592, "read": 0.592}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.794, "count": 13, "max_ms": 2.431, "p95_ms": 2.431}, "gstreamer_push": {"avg_ms": 1.322, "count": 13, "max_ms": 1.583, "p95_ms": 1.583}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.139, "count": 13, "max_ms": 53.417, "p95_ms": 53.417}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.146, "count": 13, "max_ms": 95.406, "p95_ms": 95.406}, "yolo11_640_fire_11class": {"avg_ms": 58.412, "count": 13, "max_ms": 86.328, "p95_ms": 86.328}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.027, "count": 13, "max_ms": 104.163, "p95_ms": 104.163}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.207, "count": 13, "max_ms": 114.047, "p95_ms": 114.047}, "read_interval": {"avg_ms": 2737.977, "count": 13, "max_ms": 18203.694, "p95_ms": 18203.694}, "triton_wait": {"avg_ms": 68.436, "count": 13, "max_ms": 104.175, "p95_ms": 104.175}, "visualize": {"avg_ms": 3.924, "count": 13, "max_ms": 6.521, "p95_ms": 6.521}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.963, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 47}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4294.273, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ffab8e0bc0] error while decoding MB 32 24, bytestream -17", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:11:14+09:00", "last_error_summary": "[h264 @ 0x55ffab8e0bc0] error while decoding MB 32 24, bytestream -17", "last_exit_at": "2026-07-01T15:09:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.943, "last_started_at": "2026-07-01T15:09:27+09:00", "pid": 3098, "process": {"exists": false, "pid": 3098}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.416, "sample_count": 1265, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5644, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T15:11:14+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T15:11:14+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T15:11:14.917255+09:00", "mtime_epoch": 1782886274.9172552, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206585}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:11:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 856.898, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.547, "pushed": 4.547, "read": 4.547}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.694, "count": 135, "max_ms": 12.374, "p95_ms": 8.674}, "gstreamer_push": {"avg_ms": 0.931, "count": 136, "max_ms": 2.99, "p95_ms": 1.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.336, "count": 135, "max_ms": 71.597, "p95_ms": 65.107}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.265, "count": 135, "max_ms": 103.735, "p95_ms": 88.742}, "yolo11_640_fire_11class": {"avg_ms": 52.936, "count": 135, "max_ms": 83.222, "p95_ms": 73.805}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.606, "count": 135, "max_ms": 101.034, "p95_ms": 89.058}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.385, "count": 136, "max_ms": 294.115, "p95_ms": 236.216}, "read_interval": {"avg_ms": 221.034, "count": 136, "max_ms": 303.782, "p95_ms": 245.795}, "triton_wait": {"avg_ms": 69.682, "count": 135, "max_ms": 103.749, "p95_ms": 92.824}, "visualize": {"avg_ms": 9.623, "count": 135, "max_ms": 12.252, "p95_ms": 10.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 47}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4298.605, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:11:14+09:00", "last_error_summary": "[h264 @ 0x55ffab8e0bc0] error while decoding MB 32 24, bytestream -17", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:11:14+09:00", "last_error_summary": "[h264 @ 0x55ffab8e0bc0] error while decoding MB 32 24, bytestream -17", "last_exit_at": "2026-07-01T15:11:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.148, "last_started_at": "2026-07-01T15:09:27+09:00", "pid": 3098, "process": {"exists": false, "pid": 3098}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.416, "sample_count": 1265, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5645, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:11:15+09:00", "last_frame_read_age_sec": 0.26, "last_frame_read_at": "2026-07-01T15:11:14+09:00", "mjpeg": {"age_sec": 0.251, "exists": true, "mtime": "2026-07-01T15:11:14.917255+09:00", "mtime_epoch": 1782886274.9172552, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206585}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:11:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 856.898, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.549, "pushed": 4.549, "read": 4.515}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.684, "count": 135, "max_ms": 12.374, "p95_ms": 8.657}, "gstreamer_push": {"avg_ms": 0.931, "count": 136, "max_ms": 2.99, "p95_ms": 1.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.479, "count": 135, "max_ms": 71.597, "p95_ms": 65.107}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.616, "count": 135, "max_ms": 106.901, "p95_ms": 92.384}, "yolo11_640_fire_11class": {"avg_ms": 53.195, "count": 135, "max_ms": 85.913, "p95_ms": 74.821}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.865, "count": 135, "max_ms": 101.034, "p95_ms": 89.441}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.741, "count": 136, "max_ms": 294.115, "p95_ms": 246.734}, "read_interval": {"avg_ms": 221.061, "count": 135, "max_ms": 303.782, "p95_ms": 245.795}, "triton_wait": {"avg_ms": 70.016, "count": 135, "max_ms": 106.913, "p95_ms": 93.549}, "visualize": {"avg_ms": 9.619, "count": 135, "max_ms": 12.252, "p95_ms": 10.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 47}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4298.598, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:11:14+09:00", "last_error_summary": "[h264 @ 0x55ffab8e0bc0] error while decoding MB 32 24, bytestream -17", "last_exit_at": "2026-07-01T15:11:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.151, "last_started_at": "2026-07-01T15:09:27+09:00", "pid": 3098, "process": {"exists": false, "pid": 3098}, "restart_count": 11, "restart_count_per_hour": 11, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.416, "sample_count": 1265, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5645, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:11:15+09:00", "last_frame_read_age_sec": 0.263, "last_frame_read_at": "2026-07-01T15:11:14+09:00", "mjpeg": {"age_sec": 0.254, "exists": true, "mtime": "2026-07-01T15:11:14.917255+09:00", "mtime_epoch": 1782886274.9172552, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206585}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:11:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 856.898, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.548, "pushed": 4.548, "read": 4.515}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.684, "count": 135, "max_ms": 12.374, "p95_ms": 8.657}, "gstreamer_push": {"avg_ms": 0.931, "count": 136, "max_ms": 2.99, "p95_ms": 1.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.479, "count": 135, "max_ms": 71.597, "p95_ms": 65.107}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.616, "count": 135, "max_ms": 106.901, "p95_ms": 92.384}, "yolo11_640_fire_11class": {"avg_ms": 53.195, "count": 135, "max_ms": 85.913, "p95_ms": 74.821}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.865, "count": 135, "max_ms": 101.034, "p95_ms": 89.441}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.741, "count": 136, "max_ms": 294.115, "p95_ms": 246.734}, "read_interval": {"avg_ms": 221.061, "count": 135, "max_ms": 303.782, "p95_ms": 245.795}, "triton_wait": {"avg_ms": 70.016, "count": 135, "max_ms": 106.913, "p95_ms": 93.549}, "visualize": {"avg_ms": 9.619, "count": 135, "max_ms": 12.252, "p95_ms": 10.538}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 47}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4415.367, "limit_mb": 8796093022207.996}, "detail": {"pid": 3357, "restart_count": 12, "start_count": 13}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:11:14+09:00", "last_error_summary": "[h264 @ 0x55ffab8e0bc0] error while decoding MB 32 24, bytestream -17", "last_exit_at": "2026-07-01T15:11:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:11:17+09:00", "pid": 3357, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.172, "name": "ffmpeg", "pid": 3357, "rss_mb": 141.172, "state": "S (sleeping)", "threads": 106}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.416, "sample_count": 1245, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 5645, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.495, "last_frame_pushed_at": "2026-07-01T15:11:15+09:00", "last_frame_read_age_sec": 4.751, "last_frame_read_at": "2026-07-01T15:11:14+09:00", "mjpeg": {"age_sec": 4.475, "exists": true, "mtime": "2026-07-01T15:11:15.185259+09:00", "mtime_epoch": 1782886275.1852586, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206955}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:11:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 856.898, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.834, "pushed": 3.834, "read": 3.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.652, "count": 115, "max_ms": 12.374, "p95_ms": 8.657}, "gstreamer_push": {"avg_ms": 0.935, "count": 115, "max_ms": 2.99, "p95_ms": 1.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.748, "count": 115, "max_ms": 71.597, "p95_ms": 66.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.869, "count": 115, "max_ms": 106.901, "p95_ms": 92.568}, "yolo11_640_fire_11class": {"avg_ms": 53.489, "count": 115, "max_ms": 85.913, "p95_ms": 74.844}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.19, "count": 115, "max_ms": 101.034, "p95_ms": 90.269}}, "truncated_count": 0}, "processing_total": {"avg_ms": 212.165, "count": 115, "max_ms": 294.115, "p95_ms": 251.176}, "read_interval": {"avg_ms": 221.378, "count": 115, "max_ms": 303.782, "p95_ms": 256.176}, "triton_wait": {"avg_ms": 70.314, "count": 115, "max_ms": 106.913, "p95_ms": 93.578}, "visualize": {"avg_ms": 9.597, "count": 115, "max_ms": 12.252, "p95_ms": 10.603}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 50}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4312.832, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556101f3b0c0] error while decoding MB 70 41, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:13:04+09:00", "last_error_summary": "[h264 @ 0x556101f3b0c0] error while decoding MB 70 41, bytestream -10", "last_exit_at": "2026-07-01T15:11:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 106.894, "last_started_at": "2026-07-01T15:11:17+09:00", "pid": 3357, "process": {"exists": false, "pid": 3357}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.788, "sample_count": 1263, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:13:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T15:13:04+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T15:13:04+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-01T15:13:04.514693+09:00", "mtime_epoch": 1782886384.5146933, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209461}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:13:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 860.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.27, "pushed": 4.27, "read": 4.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.379, "count": 127, "max_ms": 11.603, "p95_ms": 9.87}, "gstreamer_push": {"avg_ms": 0.977, "count": 128, "max_ms": 4.042, "p95_ms": 1.869}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.195, "count": 127, "max_ms": 72.836, "p95_ms": 65.815}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.722, "count": 127, "max_ms": 102.714, "p95_ms": 100.082}, "yolo11_640_fire_11class": {"avg_ms": 57.757, "count": 127, "max_ms": 86.338, "p95_ms": 75.479}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.037, "count": 127, "max_ms": 111.786, "p95_ms": 92.717}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.864, "count": 128, "max_ms": 330.559, "p95_ms": 268.462}, "read_interval": {"avg_ms": 235.57, "count": 128, "max_ms": 344.959, "p95_ms": 278.991}, "triton_wait": {"avg_ms": 76.854, "count": 127, "max_ms": 111.797, "p95_ms": 100.871}, "visualize": {"avg_ms": 10.156, "count": 127, "max_ms": 13.966, "p95_ms": 12.585}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 50}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4311.441, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:13:04+09:00", "last_error_summary": "[h264 @ 0x556101f3b0c0] error while decoding MB 70 41, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:13:04+09:00", "last_error_summary": "[h264 @ 0x556101f3b0c0] error while decoding MB 70 41, bytestream -10", "last_exit_at": "2026-07-01T15:13:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.038, "last_started_at": "2026-07-01T15:11:17+09:00", "pid": 3357, "process": {"exists": false, "pid": 3357}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.788, "sample_count": 1263, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:13:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:13:04+09:00", "last_frame_read_age_sec": 0.194, "last_frame_read_at": "2026-07-01T15:13:04+09:00", "mjpeg": {"age_sec": 0.182, "exists": true, "mtime": "2026-07-01T15:13:04.514693+09:00", "mtime_epoch": 1782886384.5146933, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209461}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:13:04+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 860.465, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.28, "pushed": 4.28, "read": 4.247}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.372, "count": 127, "max_ms": 11.603, "p95_ms": 9.87}, "gstreamer_push": {"avg_ms": 0.977, "count": 128, "max_ms": 4.042, "p95_ms": 1.869}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.153, "count": 127, "max_ms": 72.836, "p95_ms": 65.815}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.757, "count": 127, "max_ms": 102.714, "p95_ms": 100.082}, "yolo11_640_fire_11class": {"avg_ms": 57.735, "count": 127, "max_ms": 86.338, "p95_ms": 75.479}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.085, "count": 127, "max_ms": 111.786, "p95_ms": 92.717}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.58, "count": 128, "max_ms": 330.559, "p95_ms": 268.462}, "read_interval": {"avg_ms": 235.56, "count": 127, "max_ms": 344.959, "p95_ms": 278.991}, "triton_wait": {"avg_ms": 76.903, "count": 127, "max_ms": 111.797, "p95_ms": 100.871}, "visualize": {"avg_ms": 10.145, "count": 127, "max_ms": 13.966, "p95_ms": 12.585}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 50}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4311.559, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:13:04+09:00", "last_error_summary": "[h264 @ 0x556101f3b0c0] error while decoding MB 70 41, bytestream -10", "last_exit_at": "2026-07-01T15:13:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.042, "last_started_at": "2026-07-01T15:11:17+09:00", "pid": 3357, "process": {"exists": false, "pid": 3357}, "restart_count": 12, "restart_count_per_hour": 12, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.788, "sample_count": 1263, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:13:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:13:04+09:00", "last_frame_read_age_sec": 0.198, "last_frame_read_at": "2026-07-01T15:13:04+09:00", "mjpeg": {"age_sec": 0.186, "exists": true, "mtime": "2026-07-01T15:13:04.514693+09:00", "mtime_epoch": 1782886384.5146933, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209461}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:13:04+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 860.465, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.28, "pushed": 4.28, "read": 4.246}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.372, "count": 127, "max_ms": 11.603, "p95_ms": 9.87}, "gstreamer_push": {"avg_ms": 0.977, "count": 128, "max_ms": 4.042, "p95_ms": 1.869}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.153, "count": 127, "max_ms": 72.836, "p95_ms": 65.815}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.757, "count": 127, "max_ms": 102.714, "p95_ms": 100.082}, "yolo11_640_fire_11class": {"avg_ms": 57.735, "count": 127, "max_ms": 86.338, "p95_ms": 75.479}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.085, "count": 127, "max_ms": 111.786, "p95_ms": 92.717}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.58, "count": 128, "max_ms": 330.559, "p95_ms": 268.462}, "read_interval": {"avg_ms": 235.56, "count": 127, "max_ms": 344.959, "p95_ms": 278.991}, "triton_wait": {"avg_ms": 76.903, "count": 127, "max_ms": 111.797, "p95_ms": 100.871}, "visualize": {"avg_ms": 10.145, "count": 127, "max_ms": 13.966, "p95_ms": 12.585}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 50}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4421.055, "limit_mb": 8796093022207.996}, "detail": {"pid": 3464, "restart_count": 13, "start_count": 14}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:13:04+09:00", "last_error_summary": "[h264 @ 0x556101f3b0c0] error while decoding MB 70 41, bytestream -10", "last_exit_at": "2026-07-01T15:13:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:13:07+09:00", "pid": 3464, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.398, "name": "ffmpeg", "pid": 3464, "rss_mb": 139.398, "state": "S (sleeping)", "threads": 106}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.949, "sample_count": 1245, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:13:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.324, "last_frame_pushed_at": "2026-07-01T15:13:04+09:00", "last_frame_read_age_sec": 4.516, "last_frame_read_at": "2026-07-01T15:13:04+09:00", "mjpeg": {"age_sec": 4.308, "exists": true, "mtime": "2026-07-01T15:13:04.710696+09:00", "mtime_epoch": 1782886384.7106957, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 209350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:13:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 865.477, "name": "celery", "pid": 237, "rss_mb": 860.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.675, "pushed": 3.675, "read": 3.641}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.353, "count": 109, "max_ms": 11.603, "p95_ms": 10.405}, "gstreamer_push": {"avg_ms": 0.944, "count": 110, "max_ms": 2.996, "p95_ms": 1.272}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.307, "count": 109, "max_ms": 72.836, "p95_ms": 65.628}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.894, "count": 109, "max_ms": 102.714, "p95_ms": 100.857}, "yolo11_640_fire_11class": {"avg_ms": 57.789, "count": 109, "max_ms": 86.338, "p95_ms": 75.479}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.02, "count": 109, "max_ms": 111.786, "p95_ms": 91.877}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.102, "count": 110, "max_ms": 294.19, "p95_ms": 268.462}, "read_interval": {"avg_ms": 235.062, "count": 109, "max_ms": 304.37, "p95_ms": 278.991}, "triton_wait": {"avg_ms": 77.483, "count": 109, "max_ms": 111.797, "p95_ms": 100.983}, "visualize": {"avg_ms": 10.11, "count": 109, "max_ms": 13.026, "p95_ms": 12.548}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 55}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4029.082, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55965060e940] error while decoding MB 59 18, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:15:13+09:00", "last_error_summary": "[h264 @ 0x55965060e940] error while decoding MB 59 18, bytestream -5", "last_exit_at": "2026-07-01T15:13:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.697, "last_started_at": "2026-07-01T15:13:07+09:00", "pid": 3464, "process": {"exists": false, "pid": 3464}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.646, "sample_count": 1298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6667, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:15:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T15:15:13+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T15:15:13+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T15:15:13.668460+09:00", "mtime_epoch": 1782886513.6684604, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203242}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:15:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.109, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.597, "pushed": 4.597, "read": 4.597}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.799, "count": 137, "max_ms": 9.406, "p95_ms": 8.812}, "gstreamer_push": {"avg_ms": 0.918, "count": 137, "max_ms": 2.896, "p95_ms": 1.033}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.174, "count": 137, "max_ms": 70.81, "p95_ms": 62.936}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.293, "count": 137, "max_ms": 89.324, "p95_ms": 86.441}, "yolo11_640_fire_11class": {"avg_ms": 56.313, "count": 137, "max_ms": 74.145, "p95_ms": 72.923}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.646, "count": 137, "max_ms": 93.571, "p95_ms": 89.333}}, "truncated_count": 0}, "processing_total": {"avg_ms": 209.414, "count": 137, "max_ms": 255.109, "p95_ms": 239.048}, "read_interval": {"avg_ms": 218.671, "count": 137, "max_ms": 266.819, "p95_ms": 249.345}, "triton_wait": {"avg_ms": 69.893, "count": 137, "max_ms": 93.583, "p95_ms": 89.342}, "visualize": {"avg_ms": 9.644, "count": 137, "max_ms": 11.642, "p95_ms": 11.122}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.799, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 55}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4028.223, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:15:13+09:00", "last_error_summary": "[h264 @ 0x55965060e940] error while decoding MB 59 18, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:15:13+09:00", "last_error_summary": "[h264 @ 0x55965060e940] error while decoding MB 59 18, bytestream -5", "last_exit_at": "2026-07-01T15:15:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.83, "last_started_at": "2026-07-01T15:13:07+09:00", "pid": 3464, "process": {"exists": false, "pid": 3464}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.646, "sample_count": 1299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:15:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:15:13+09:00", "last_frame_read_age_sec": 0.186, "last_frame_read_at": "2026-07-01T15:15:13+09:00", "mjpeg": {"age_sec": 0.179, "exists": true, "mtime": "2026-07-01T15:15:13.668460+09:00", "mtime_epoch": 1782886513.6684604, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203242}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:15:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.109, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.61, "pushed": 4.61, "read": 4.577}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.823, "count": 137, "max_ms": 11.212, "p95_ms": 8.845}, "gstreamer_push": {"avg_ms": 0.917, "count": 138, "max_ms": 2.896, "p95_ms": 1.033}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.027, "count": 137, "max_ms": 70.81, "p95_ms": 62.936}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.224, "count": 137, "max_ms": 89.324, "p95_ms": 86.441}, "yolo11_640_fire_11class": {"avg_ms": 56.227, "count": 137, "max_ms": 74.145, "p95_ms": 72.923}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.559, "count": 137, "max_ms": 93.571, "p95_ms": 89.333}}, "truncated_count": 0}, "processing_total": {"avg_ms": 209.213, "count": 138, "max_ms": 255.109, "p95_ms": 239.048}, "read_interval": {"avg_ms": 218.671, "count": 137, "max_ms": 266.819, "p95_ms": 249.345}, "triton_wait": {"avg_ms": 69.806, "count": 137, "max_ms": 93.583, "p95_ms": 89.342}, "visualize": {"avg_ms": 9.641, "count": 137, "max_ms": 11.642, "p95_ms": 11.122}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 55}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4028.215, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:15:13+09:00", "last_error_summary": "[h264 @ 0x55965060e940] error while decoding MB 59 18, bytestream -5", "last_exit_at": "2026-07-01T15:15:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.834, "last_started_at": "2026-07-01T15:13:07+09:00", "pid": 3464, "process": {"exists": false, "pid": 3464}, "restart_count": 13, "restart_count_per_hour": 13, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.646, "sample_count": 1299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:15:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:15:13+09:00", "last_frame_read_age_sec": 0.189, "last_frame_read_at": "2026-07-01T15:15:13+09:00", "mjpeg": {"age_sec": 0.183, "exists": true, "mtime": "2026-07-01T15:15:13.668460+09:00", "mtime_epoch": 1782886513.6684604, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203242}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:15:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.109, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.61, "pushed": 4.61, "read": 4.576}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.823, "count": 137, "max_ms": 11.212, "p95_ms": 8.845}, "gstreamer_push": {"avg_ms": 0.917, "count": 138, "max_ms": 2.896, "p95_ms": 1.033}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.027, "count": 137, "max_ms": 70.81, "p95_ms": 62.936}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.224, "count": 137, "max_ms": 89.324, "p95_ms": 86.441}, "yolo11_640_fire_11class": {"avg_ms": 56.227, "count": 137, "max_ms": 74.145, "p95_ms": 72.923}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.559, "count": 137, "max_ms": 93.571, "p95_ms": 89.333}}, "truncated_count": 0}, "processing_total": {"avg_ms": 209.213, "count": 138, "max_ms": 255.109, "p95_ms": 239.048}, "read_interval": {"avg_ms": 218.671, "count": 137, "max_ms": 266.819, "p95_ms": 249.345}, "triton_wait": {"avg_ms": 69.806, "count": 137, "max_ms": 93.583, "p95_ms": 89.342}, "visualize": {"avg_ms": 9.641, "count": 137, "max_ms": 11.642, "p95_ms": 11.122}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 55}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4133.812, "limit_mb": 8796093022207.996}, "detail": {"pid": 3571, "restart_count": 14, "start_count": 15}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:15:13+09:00", "last_error_summary": "[h264 @ 0x55965060e940] error while decoding MB 59 18, bytestream -5", "last_exit_at": "2026-07-01T15:15:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:15:16+09:00", "pid": 3571, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.539, "name": "ffmpeg", "pid": 3571, "rss_mb": 136.539, "state": "S (sleeping)", "threads": 106}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.651, "sample_count": 1280, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 6668, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:15:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.267, "last_frame_pushed_at": "2026-07-01T15:15:13+09:00", "last_frame_read_age_sec": 4.45, "last_frame_read_at": "2026-07-01T15:15:13+09:00", "mjpeg": {"age_sec": 4.252, "exists": true, "mtime": "2026-07-01T15:15:13.860463+09:00", "mtime_epoch": 1782886513.8604631, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203674}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:15:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.109, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.98, "pushed": 3.98, "read": 3.946}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.755, "count": 118, "max_ms": 11.212, "p95_ms": 8.845}, "gstreamer_push": {"avg_ms": 0.917, "count": 119, "max_ms": 2.896, "p95_ms": 1.033}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.986, "count": 118, "max_ms": 70.81, "p95_ms": 62.936}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.054, "count": 118, "max_ms": 89.324, "p95_ms": 87.452}, "yolo11_640_fire_11class": {"avg_ms": 56.127, "count": 118, "max_ms": 74.145, "p95_ms": 72.923}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.341, "count": 118, "max_ms": 93.091, "p95_ms": 89.188}}, "truncated_count": 0}, "processing_total": {"avg_ms": 207.941, "count": 119, "max_ms": 255.109, "p95_ms": 240.4}, "read_interval": {"avg_ms": 217.359, "count": 118, "max_ms": 266.819, "p95_ms": 250.99}, "triton_wait": {"avg_ms": 69.626, "count": 118, "max_ms": 93.103, "p95_ms": 89.335}, "visualize": {"avg_ms": 9.589, "count": 118, "max_ms": 11.642, "p95_ms": 11.122}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 59}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3987.625, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c00a883c80] error while decoding MB 25 33, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:17:03+09:00", "last_error_summary": "[h264 @ 0x55c00a883c80] error while decoding MB 25 33, bytestream -14", "last_exit_at": "2026-07-01T15:15:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.453, "last_started_at": "2026-07-01T15:15:16+09:00", "pid": 3571, "process": {"exists": false, "pid": 3571}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.466, "sample_count": 1309, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7139, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:17:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-01T15:17:03+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T15:17:03+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-01T15:17:03.514013+09:00", "mtime_epoch": 1782886623.5140128, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198739}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:17:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.574, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.534, "pushed": 4.534, "read": 4.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.875, "count": 136, "max_ms": 12.298, "p95_ms": 9.034}, "gstreamer_push": {"avg_ms": 0.938, "count": 136, "max_ms": 2.837, "p95_ms": 1.803}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.806, "count": 136, "max_ms": 71.205, "p95_ms": 63.246}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.261, "count": 136, "max_ms": 89.813, "p95_ms": 85.469}, "yolo11_640_fire_11class": {"avg_ms": 54.033, "count": 136, "max_ms": 75.853, "p95_ms": 72.627}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.863, "count": 136, "max_ms": 94.326, "p95_ms": 89.938}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.588, "count": 136, "max_ms": 286.366, "p95_ms": 237.878}, "read_interval": {"avg_ms": 220.29, "count": 137, "max_ms": 297.733, "p95_ms": 248.32}, "triton_wait": {"avg_ms": 67.008, "count": 136, "max_ms": 94.338, "p95_ms": 89.95}, "visualize": {"avg_ms": 9.866, "count": 136, "max_ms": 14.354, "p95_ms": 11.181}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 59}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3986.773, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:17:03+09:00", "last_error_summary": "[h264 @ 0x55c00a883c80] error while decoding MB 25 33, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:17:03+09:00", "last_error_summary": "[h264 @ 0x55c00a883c80] error while decoding MB 25 33, bytestream -14", "last_exit_at": "2026-07-01T15:17:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.619, "last_started_at": "2026-07-01T15:15:16+09:00", "pid": 3571, "process": {"exists": false, "pid": 3571}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.466, "sample_count": 1310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7140, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:17:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:17:03+09:00", "last_frame_read_age_sec": 0.221, "last_frame_read_at": "2026-07-01T15:17:03+09:00", "mjpeg": {"age_sec": 0.217, "exists": true, "mtime": "2026-07-01T15:17:03.514013+09:00", "mtime_epoch": 1782886623.5140128, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198739}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:17:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.574, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.574, "pushed": 4.574, "read": 4.54}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.873, "count": 136, "max_ms": 12.298, "p95_ms": 9.034}, "gstreamer_push": {"avg_ms": 0.937, "count": 137, "max_ms": 2.837, "p95_ms": 1.803}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.782, "count": 136, "max_ms": 71.205, "p95_ms": 63.246}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.279, "count": 136, "max_ms": 89.813, "p95_ms": 85.469}, "yolo11_640_fire_11class": {"avg_ms": 54.071, "count": 136, "max_ms": 75.853, "p95_ms": 72.627}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.887, "count": 136, "max_ms": 94.326, "p95_ms": 89.938}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.634, "count": 137, "max_ms": 286.366, "p95_ms": 237.878}, "read_interval": {"avg_ms": 220.139, "count": 136, "max_ms": 297.733, "p95_ms": 248.32}, "triton_wait": {"avg_ms": 67.032, "count": 136, "max_ms": 94.338, "p95_ms": 89.95}, "visualize": {"avg_ms": 9.866, "count": 136, "max_ms": 14.354, "p95_ms": 11.181}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 59}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3986.703, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:17:03+09:00", "last_error_summary": "[h264 @ 0x55c00a883c80] error while decoding MB 25 33, bytestream -14", "last_exit_at": "2026-07-01T15:17:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.623, "last_started_at": "2026-07-01T15:15:16+09:00", "pid": 3571, "process": {"exists": false, "pid": 3571}, "restart_count": 14, "restart_count_per_hour": 14, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.466, "sample_count": 1310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7140, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:17:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:17:03+09:00", "last_frame_read_age_sec": 0.225, "last_frame_read_at": "2026-07-01T15:17:03+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-01T15:17:03.514013+09:00", "mtime_epoch": 1782886623.5140128, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198739}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:17:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.574, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.573, "pushed": 4.573, "read": 4.54}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.873, "count": 136, "max_ms": 12.298, "p95_ms": 9.034}, "gstreamer_push": {"avg_ms": 0.937, "count": 137, "max_ms": 2.837, "p95_ms": 1.803}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.782, "count": 136, "max_ms": 71.205, "p95_ms": 63.246}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.279, "count": 136, "max_ms": 89.813, "p95_ms": 85.469}, "yolo11_640_fire_11class": {"avg_ms": 54.071, "count": 136, "max_ms": 75.853, "p95_ms": 72.627}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.887, "count": 136, "max_ms": 94.326, "p95_ms": 89.938}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.634, "count": 137, "max_ms": 286.366, "p95_ms": 237.878}, "read_interval": {"avg_ms": 220.139, "count": 136, "max_ms": 297.733, "p95_ms": 248.32}, "triton_wait": {"avg_ms": 67.032, "count": 136, "max_ms": 94.338, "p95_ms": 89.95}, "visualize": {"avg_ms": 9.866, "count": 136, "max_ms": 14.354, "p95_ms": 11.181}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 59}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4092.129, "limit_mb": 8796093022207.996}, "detail": {"pid": 3678, "restart_count": 15, "start_count": 16}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:17:03+09:00", "last_error_summary": "[h264 @ 0x55c00a883c80] error while decoding MB 25 33, bytestream -14", "last_exit_at": "2026-07-01T15:17:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:17:06+09:00", "pid": 3678, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.512, "name": "ffmpeg", "pid": 3678, "rss_mb": 136.512, "state": "S (sleeping)", "threads": 106}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 248.803, "sample_count": 1291, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7140, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:17:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.544, "last_frame_pushed_at": "2026-07-01T15:17:03+09:00", "last_frame_read_age_sec": 4.762, "last_frame_read_at": "2026-07-01T15:17:03+09:00", "mjpeg": {"age_sec": 4.526, "exists": true, "mtime": "2026-07-01T15:17:03.746016+09:00", "mtime_epoch": 1782886623.746016, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194497}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:17:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.574, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.885, "pushed": 3.885, "read": 3.851}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.899, "count": 116, "max_ms": 12.298, "p95_ms": 9.247}, "gstreamer_push": {"avg_ms": 0.946, "count": 116, "max_ms": 2.837, "p95_ms": 1.84}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.4, "count": 116, "max_ms": 71.205, "p95_ms": 63.161}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.964, "count": 116, "max_ms": 89.813, "p95_ms": 85.337}, "yolo11_640_fire_11class": {"avg_ms": 53.726, "count": 116, "max_ms": 75.853, "p95_ms": 72.627}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.325, "count": 116, "max_ms": 94.326, "p95_ms": 88.349}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.51, "count": 116, "max_ms": 286.366, "p95_ms": 244.233}, "read_interval": {"avg_ms": 219.999, "count": 115, "max_ms": 297.733, "p95_ms": 253.476}, "triton_wait": {"avg_ms": 66.494, "count": 116, "max_ms": 94.338, "p95_ms": 88.902}, "visualize": {"avg_ms": 9.887, "count": 116, "max_ms": 14.354, "p95_ms": 11.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.859, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 64}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3875.5, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5637cbce7140] error while decoding MB 30 20, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:19:14+09:00", "last_error_summary": "[h264 @ 0x5637cbce7140] error while decoding MB 30 20, bytestream -10", "last_exit_at": "2026-07-01T15:17:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.963, "last_started_at": "2026-07-01T15:17:06+09:00", "pid": 3678, "process": {"exists": false, "pid": 3678}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.006, "sample_count": 1310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7706, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:19:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T15:19:14+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T15:19:14+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T15:19:14.191906+09:00", "mtime_epoch": 1782886754.191906, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205579}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:19:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.407, "pushed": 4.407, "read": 4.407}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.289, "count": 131, "max_ms": 14.558, "p95_ms": 9.903}, "gstreamer_push": {"avg_ms": 0.901, "count": 132, "max_ms": 2.96, "p95_ms": 0.955}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.646, "count": 131, "max_ms": 69.635, "p95_ms": 63.547}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.384, "count": 131, "max_ms": 88.915, "p95_ms": 85.333}, "yolo11_640_fire_11class": {"avg_ms": 55.619, "count": 131, "max_ms": 74.526, "p95_ms": 68.359}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.014, "count": 131, "max_ms": 91.222, "p95_ms": 86.932}}, "truncated_count": 0}, "processing_total": {"avg_ms": 218.346, "count": 132, "max_ms": 290.052, "p95_ms": 237.39}, "read_interval": {"avg_ms": 228.129, "count": 132, "max_ms": 301.51, "p95_ms": 247.078}, "triton_wait": {"avg_ms": 69.458, "count": 131, "max_ms": 91.234, "p95_ms": 87.612}, "visualize": {"avg_ms": 10.137, "count": 131, "max_ms": 13.594, "p95_ms": 12.218}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 64}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3874.527, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:19:14+09:00", "last_error_summary": "[h264 @ 0x5637cbce7140] error while decoding MB 30 20, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:19:14+09:00", "last_error_summary": "[h264 @ 0x5637cbce7140] error while decoding MB 30 20, bytestream -10", "last_exit_at": "2026-07-01T15:19:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.143, "last_started_at": "2026-07-01T15:17:06+09:00", "pid": 3678, "process": {"exists": false, "pid": 3678}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.929, "sample_count": 1310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7707, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:19:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T15:19:14+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T15:19:14+09:00", "mjpeg": {"age_sec": 0.221, "exists": true, "mtime": "2026-07-01T15:19:14.191906+09:00", "mtime_epoch": 1782886754.191906, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205579}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:19:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.816, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.415, "pushed": 4.415, "read": 4.381}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.275, "count": 131, "max_ms": 14.558, "p95_ms": 9.808}, "gstreamer_push": {"avg_ms": 0.902, "count": 132, "max_ms": 2.96, "p95_ms": 0.955}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.678, "count": 131, "max_ms": 69.635, "p95_ms": 63.547}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.412, "count": 131, "max_ms": 88.915, "p95_ms": 85.333}, "yolo11_640_fire_11class": {"avg_ms": 55.629, "count": 131, "max_ms": 74.526, "p95_ms": 68.359}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.023, "count": 131, "max_ms": 91.222, "p95_ms": 86.932}}, "truncated_count": 0}, "processing_total": {"avg_ms": 218.419, "count": 132, "max_ms": 290.052, "p95_ms": 237.39}, "read_interval": {"avg_ms": 228.151, "count": 131, "max_ms": 301.51, "p95_ms": 247.078}, "triton_wait": {"avg_ms": 69.468, "count": 131, "max_ms": 91.234, "p95_ms": 87.612}, "visualize": {"avg_ms": 10.114, "count": 131, "max_ms": 13.324, "p95_ms": 12.092}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 64}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3874.52, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:19:14+09:00", "last_error_summary": "[h264 @ 0x5637cbce7140] error while decoding MB 30 20, bytestream -10", "last_exit_at": "2026-07-01T15:19:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.146, "last_started_at": "2026-07-01T15:17:06+09:00", "pid": 3678, "process": {"exists": false, "pid": 3678}, "restart_count": 15, "restart_count_per_hour": 15, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.929, "sample_count": 1310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7707, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:19:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T15:19:14+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T15:19:14+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T15:19:14.191906+09:00", "mtime_epoch": 1782886754.191906, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205579}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:19:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.816, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.414, "pushed": 4.414, "read": 4.381}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.275, "count": 131, "max_ms": 14.558, "p95_ms": 9.808}, "gstreamer_push": {"avg_ms": 0.902, "count": 132, "max_ms": 2.96, "p95_ms": 0.955}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.678, "count": 131, "max_ms": 69.635, "p95_ms": 63.547}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.412, "count": 131, "max_ms": 88.915, "p95_ms": 85.333}, "yolo11_640_fire_11class": {"avg_ms": 55.629, "count": 131, "max_ms": 74.526, "p95_ms": 68.359}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.023, "count": 131, "max_ms": 91.222, "p95_ms": 86.932}}, "truncated_count": 0}, "processing_total": {"avg_ms": 218.419, "count": 132, "max_ms": 290.052, "p95_ms": 237.39}, "read_interval": {"avg_ms": 228.151, "count": 131, "max_ms": 301.51, "p95_ms": 247.078}, "triton_wait": {"avg_ms": 69.468, "count": 131, "max_ms": 91.234, "p95_ms": 87.612}, "visualize": {"avg_ms": 10.114, "count": 131, "max_ms": 13.324, "p95_ms": 12.092}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 64}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3972.559, "limit_mb": 8796093022207.996}, "detail": {"pid": 3785, "restart_count": 16, "start_count": 17}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:19:14+09:00", "last_error_summary": "[h264 @ 0x5637cbce7140] error while decoding MB 30 20, bytestream -10", "last_exit_at": "2026-07-01T15:19:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:19:16+09:00", "pid": 3785, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.203, "name": "ffmpeg", "pid": 3785, "rss_mb": 139.203, "state": "S (sleeping)", "threads": 106}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.76, "sample_count": 1291, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 7707, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:19:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.341, "last_frame_pushed_at": "2026-07-01T15:19:14+09:00", "last_frame_read_age_sec": 4.568, "last_frame_read_at": "2026-07-01T15:19:14+09:00", "mjpeg": {"age_sec": 4.326, "exists": true, "mtime": "2026-07-01T15:19:14.423909+09:00", "mtime_epoch": 1782886754.4239092, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:19:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 858.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.775, "pushed": 3.775, "read": 3.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.343, "count": 112, "max_ms": 14.558, "p95_ms": 9.808}, "gstreamer_push": {"avg_ms": 0.908, "count": 113, "max_ms": 2.96, "p95_ms": 0.955}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.305, "count": 112, "max_ms": 68.081, "p95_ms": 63.058}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.36, "count": 112, "max_ms": 88.915, "p95_ms": 85.333}, "yolo11_640_fire_11class": {"avg_ms": 55.374, "count": 112, "max_ms": 74.526, "p95_ms": 68.359}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.892, "count": 112, "max_ms": 89.68, "p95_ms": 86.384}}, "truncated_count": 0}, "processing_total": {"avg_ms": 218.681, "count": 113, "max_ms": 290.052, "p95_ms": 238.708}, "read_interval": {"avg_ms": 228.408, "count": 112, "max_ms": 301.51, "p95_ms": 248.001}, "triton_wait": {"avg_ms": 69.33, "count": 112, "max_ms": 89.692, "p95_ms": 86.395}, "visualize": {"avg_ms": 10.109, "count": 112, "max_ms": 12.717, "p95_ms": 11.189}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 68}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3873.797, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555981bfaac0] error while decoding MB 48 5, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:21:21+09:00", "last_error_summary": "[h264 @ 0x555981bfaac0] error while decoding MB 48 5, bytestream -5", "last_exit_at": "2026-07-01T15:19:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.799, "last_started_at": "2026-07-01T15:19:16+09:00", "pid": 3785, "process": {"exists": false, "pid": 3785}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.76, "sample_count": 1304, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8253, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:21:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-01T15:21:21+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-01T15:21:21+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T15:21:21.501788+09:00", "mtime_epoch": 1782886881.5017884, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206307}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:21:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.383, "pushed": 4.383, "read": 4.383}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.258, "count": 130, "max_ms": 10.633, "p95_ms": 8.902}, "gstreamer_push": {"avg_ms": 0.899, "count": 131, "max_ms": 1.811, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.041, "count": 130, "max_ms": 79.451, "p95_ms": 66.194}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.585, "count": 130, "max_ms": 95.293, "p95_ms": 86.009}, "yolo11_640_fire_11class": {"avg_ms": 54.853, "count": 130, "max_ms": 79.464, "p95_ms": 68.79}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.335, "count": 130, "max_ms": 101.866, "p95_ms": 89.687}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.778, "count": 131, "max_ms": 282.342, "p95_ms": 250.717}, "read_interval": {"avg_ms": 229.402, "count": 131, "max_ms": 291.925, "p95_ms": 261.07}, "triton_wait": {"avg_ms": 69.526, "count": 130, "max_ms": 101.879, "p95_ms": 90.811}, "visualize": {"avg_ms": 10.229, "count": 130, "max_ms": 14.491, "p95_ms": 11.262}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 68}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3873.301, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:21:21+09:00", "last_error_summary": "[h264 @ 0x555981bfaac0] error while decoding MB 48 5, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:21:21+09:00", "last_error_summary": "[h264 @ 0x555981bfaac0] error while decoding MB 48 5, bytestream -5", "last_exit_at": "2026-07-01T15:21:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.954, "last_started_at": "2026-07-01T15:19:16+09:00", "pid": 3785, "process": {"exists": false, "pid": 3785}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.76, "sample_count": 1304, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8254, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:21:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:21:21+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-01T15:21:21+09:00", "mjpeg": {"age_sec": 0.2, "exists": true, "mtime": "2026-07-01T15:21:21.501788+09:00", "mtime_epoch": 1782886881.5017884, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206307}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:21:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.426, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.392, "pushed": 4.392, "read": 4.358}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.269, "count": 131, "max_ms": 10.633, "p95_ms": 8.921}, "gstreamer_push": {"avg_ms": 0.898, "count": 131, "max_ms": 1.811, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.001, "count": 131, "max_ms": 79.451, "p95_ms": 66.194}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.538, "count": 131, "max_ms": 95.293, "p95_ms": 86.009}, "yolo11_640_fire_11class": {"avg_ms": 54.823, "count": 131, "max_ms": 79.464, "p95_ms": 68.79}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.291, "count": 131, "max_ms": 101.866, "p95_ms": 89.687}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.696, "count": 131, "max_ms": 282.342, "p95_ms": 250.717}, "read_interval": {"avg_ms": 229.428, "count": 130, "max_ms": 291.925, "p95_ms": 261.07}, "triton_wait": {"avg_ms": 69.48, "count": 131, "max_ms": 101.879, "p95_ms": 90.811}, "visualize": {"avg_ms": 10.224, "count": 131, "max_ms": 14.491, "p95_ms": 11.262}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.828, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 68}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3873.293, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:21:21+09:00", "last_error_summary": "[h264 @ 0x555981bfaac0] error while decoding MB 48 5, bytestream -5", "last_exit_at": "2026-07-01T15:21:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.957, "last_started_at": "2026-07-01T15:19:16+09:00", "pid": 3785, "process": {"exists": false, "pid": 3785}, "restart_count": 16, "restart_count_per_hour": 16, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.76, "sample_count": 1304, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8254, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:21:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:21:21+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T15:21:21+09:00", "mjpeg": {"age_sec": 0.204, "exists": true, "mtime": "2026-07-01T15:21:21.501788+09:00", "mtime_epoch": 1782886881.5017884, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206307}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:21:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.426, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.391, "pushed": 4.391, "read": 4.358}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.269, "count": 131, "max_ms": 10.633, "p95_ms": 8.921}, "gstreamer_push": {"avg_ms": 0.898, "count": 131, "max_ms": 1.811, "p95_ms": 1.226}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.001, "count": 131, "max_ms": 79.451, "p95_ms": 66.194}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.538, "count": 131, "max_ms": 95.293, "p95_ms": 86.009}, "yolo11_640_fire_11class": {"avg_ms": 54.823, "count": 131, "max_ms": 79.464, "p95_ms": 68.79}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.291, "count": 131, "max_ms": 101.866, "p95_ms": 89.687}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.696, "count": 131, "max_ms": 282.342, "p95_ms": 250.717}, "read_interval": {"avg_ms": 229.428, "count": 130, "max_ms": 291.925, "p95_ms": 261.07}, "triton_wait": {"avg_ms": 69.48, "count": 131, "max_ms": 101.879, "p95_ms": 90.811}, "visualize": {"avg_ms": 10.224, "count": 131, "max_ms": 14.491, "p95_ms": 11.262}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.831, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 68}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3984.254, "limit_mb": 8796093022207.996}, "detail": {"pid": 3892, "restart_count": 17, "start_count": 18}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:21:21+09:00", "last_error_summary": "[h264 @ 0x555981bfaac0] error while decoding MB 48 5, bytestream -5", "last_exit_at": "2026-07-01T15:21:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:21:24+09:00", "pid": 3892, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.953, "name": "ffmpeg", "pid": 3892, "rss_mb": 141.953, "state": "S (sleeping)", "threads": 106}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.93, "sample_count": 1282, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8254, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:21:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 5.01, "last_frame_pushed_at": "2026-07-01T15:21:21+09:00", "last_frame_read_age_sec": 5.218, "last_frame_read_at": "2026-07-01T15:21:21+09:00", "mjpeg": {"age_sec": 5.0, "exists": true, "mtime": "2026-07-01T15:21:21.709791+09:00", "mtime_epoch": 1782886881.7097914, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199168}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:21:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.653, "pushed": 3.653, "read": 3.619}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.247, "count": 109, "max_ms": 10.633, "p95_ms": 8.902}, "gstreamer_push": {"avg_ms": 0.889, "count": 109, "max_ms": 1.806, "p95_ms": 0.976}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.848, "count": 109, "max_ms": 69.863, "p95_ms": 66.194}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.671, "count": 109, "max_ms": 91.982, "p95_ms": 86.009}, "yolo11_640_fire_11class": {"avg_ms": 54.62, "count": 109, "max_ms": 74.824, "p95_ms": 67.781}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.314, "count": 109, "max_ms": 98.762, "p95_ms": 89.687}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.37, "count": 109, "max_ms": 282.342, "p95_ms": 251.731}, "read_interval": {"avg_ms": 230.097, "count": 108, "max_ms": 291.925, "p95_ms": 261.259}, "triton_wait": {"avg_ms": 69.539, "count": 109, "max_ms": 98.774, "p95_ms": 90.811}, "visualize": {"avg_ms": 10.288, "count": 109, "max_ms": 14.491, "p95_ms": 11.262}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.84, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_152217_922da91f.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 4214.539, "limit_mb": 8796093022207.996}, "detail": {"pid": 4002, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:22:17+09:00", "pid": 4002, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.809, "name": "ffmpeg", "pid": 4002, "rss_mb": 137.809, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.795}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "922da91f", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 544.838, "exists": true, "mtime": "2026-07-01T15:13:14.690830+09:00", "mtime_epoch": 1782886394.6908298, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 258889}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T15:22:19+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 709.395, "name": "celery", "pid": 239, "rss_mb": 678.949, "state": "R (running)", "threads": 118}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T15:22:17+09:00", "state": "running", "task_id": "462d5f3d-bdb2-4345-83e1-bfb84e3b960e", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 73}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4144.133, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55764adbef00] error while decoding MB 61 12, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:23:33+09:00", "last_error_summary": "[h264 @ 0x55764adbef00] error while decoding MB 61 12, bytestream -5", "last_exit_at": "2026-07-01T15:21:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.284, "last_started_at": "2026-07-01T15:21:24+09:00", "pid": 3892, "process": {"exists": false, "pid": 3892}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.304, "sample_count": 1303, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8832, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:23:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T15:23:33+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T15:23:33+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T15:23:33.951778+09:00", "mtime_epoch": 1782887013.9517777, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208678}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:23:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.238, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.494, "pushed": 4.494, "read": 4.494}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.779, "count": 134, "max_ms": 10.486, "p95_ms": 9.299}, "gstreamer_push": {"avg_ms": 0.923, "count": 134, "max_ms": 3.136, "p95_ms": 0.964}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.582, "count": 134, "max_ms": 78.862, "p95_ms": 72.653}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.403, "count": 134, "max_ms": 121.48, "p95_ms": 101.987}, "yolo11_640_fire_11class": {"avg_ms": 56.916, "count": 134, "max_ms": 82.751, "p95_ms": 77.803}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.534, "count": 134, "max_ms": 105.163, "p95_ms": 100.864}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.908, "count": 134, "max_ms": 291.686, "p95_ms": 249.863}, "read_interval": {"avg_ms": 223.683, "count": 134, "max_ms": 300.825, "p95_ms": 259.689}, "triton_wait": {"avg_ms": 76.263, "count": 134, "max_ms": 121.493, "p95_ms": 103.533}, "visualize": {"avg_ms": 9.642, "count": 134, "max_ms": 12.702, "p95_ms": 10.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.817, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 73}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4143.074, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:23:33+09:00", "last_error_summary": "[h264 @ 0x55764adbef00] error while decoding MB 61 12, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:23:33+09:00", "last_error_summary": "[h264 @ 0x55764adbef00] error while decoding MB 61 12, bytestream -5", "last_exit_at": "2026-07-01T15:23:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.431, "last_started_at": "2026-07-01T15:21:24+09:00", "pid": 3892, "process": {"exists": false, "pid": 3892}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.304, "sample_count": 1304, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8833, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:23:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:23:34+09:00", "last_frame_read_age_sec": 0.2, "last_frame_read_at": "2026-07-01T15:23:33+09:00", "mjpeg": {"age_sec": 0.188, "exists": true, "mtime": "2026-07-01T15:23:33.951778+09:00", "mtime_epoch": 1782887013.9517777, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208678}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:23:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.238, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.505, "pushed": 4.505, "read": 4.472}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.775, "count": 134, "max_ms": 10.486, "p95_ms": 9.299}, "gstreamer_push": {"avg_ms": 0.924, "count": 135, "max_ms": 3.136, "p95_ms": 1.052}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.518, "count": 134, "max_ms": 78.862, "p95_ms": 72.653}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.389, "count": 134, "max_ms": 121.48, "p95_ms": 101.987}, "yolo11_640_fire_11class": {"avg_ms": 56.901, "count": 134, "max_ms": 82.751, "p95_ms": 77.803}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.512, "count": 134, "max_ms": 105.163, "p95_ms": 100.864}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.752, "count": 135, "max_ms": 291.686, "p95_ms": 249.863}, "read_interval": {"avg_ms": 223.683, "count": 134, "max_ms": 300.825, "p95_ms": 259.689}, "triton_wait": {"avg_ms": 76.24, "count": 134, "max_ms": 121.493, "p95_ms": 103.533}, "visualize": {"avg_ms": 9.631, "count": 134, "max_ms": 12.702, "p95_ms": 10.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 73}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4143.066, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:23:33+09:00", "last_error_summary": "[h264 @ 0x55764adbef00] error while decoding MB 61 12, bytestream -5", "last_exit_at": "2026-07-01T15:23:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.435, "last_started_at": "2026-07-01T15:21:24+09:00", "pid": 3892, "process": {"exists": false, "pid": 3892}, "restart_count": 17, "restart_count_per_hour": 17, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.304, "sample_count": 1304, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8833, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:23:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T15:23:34+09:00", "last_frame_read_age_sec": 0.203, "last_frame_read_at": "2026-07-01T15:23:33+09:00", "mjpeg": {"age_sec": 0.192, "exists": true, "mtime": "2026-07-01T15:23:33.951778+09:00", "mtime_epoch": 1782887013.9517777, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208678}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:23:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.238, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.505, "pushed": 4.505, "read": 4.472}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.775, "count": 134, "max_ms": 10.486, "p95_ms": 9.299}, "gstreamer_push": {"avg_ms": 0.924, "count": 135, "max_ms": 3.136, "p95_ms": 1.052}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.518, "count": 134, "max_ms": 78.862, "p95_ms": 72.653}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.389, "count": 134, "max_ms": 121.48, "p95_ms": 101.987}, "yolo11_640_fire_11class": {"avg_ms": 56.901, "count": 134, "max_ms": 82.751, "p95_ms": 77.803}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.512, "count": 134, "max_ms": 105.163, "p95_ms": 100.864}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.752, "count": 135, "max_ms": 291.686, "p95_ms": 249.863}, "read_interval": {"avg_ms": 223.683, "count": 134, "max_ms": 300.825, "p95_ms": 259.689}, "triton_wait": {"avg_ms": 76.24, "count": 134, "max_ms": 121.493, "p95_ms": 103.533}, "visualize": {"avg_ms": 9.631, "count": 134, "max_ms": 12.702, "p95_ms": 10.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 73}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4157.602, "limit_mb": 8796093022207.996}, "detail": {"pid": 4155, "restart_count": 18, "start_count": 19}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:23:33+09:00", "last_error_summary": "[h264 @ 0x55764adbef00] error while decoding MB 61 12, bytestream -5", "last_exit_at": "2026-07-01T15:23:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:23:36+09:00", "pid": 4155, "process": {"exists": true, "fd_count": 4, "hwm_mb": 52.699, "name": "ffmpeg", "pid": 4155, "rss_mb": 52.699, "state": "S (sleeping)", "threads": 1}, "restart_count": 18, "restart_count_per_hour": 18, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.304, "sample_count": 1284, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 8833, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:23:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.427, "last_frame_pushed_at": "2026-07-01T15:23:34+09:00", "last_frame_read_age_sec": 4.622, "last_frame_read_at": "2026-07-01T15:23:33+09:00", "mjpeg": {"age_sec": 4.406, "exists": true, "mtime": "2026-07-01T15:23:34.155781+09:00", "mtime_epoch": 1782887014.1557808, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208838}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:23:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.238, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.884, "pushed": 3.884, "read": 3.851}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.706, "count": 116, "max_ms": 10.486, "p95_ms": 9.209}, "gstreamer_push": {"avg_ms": 0.932, "count": 116, "max_ms": 3.136, "p95_ms": 1.071}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.039, "count": 116, "max_ms": 78.862, "p95_ms": 70.746}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.312, "count": 116, "max_ms": 121.48, "p95_ms": 101.917}, "yolo11_640_fire_11class": {"avg_ms": 56.385, "count": 116, "max_ms": 82.751, "p95_ms": 77.837}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.646, "count": 116, "max_ms": 105.163, "p95_ms": 102.141}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.575, "count": 116, "max_ms": 277.091, "p95_ms": 249.173}, "read_interval": {"avg_ms": 221.537, "count": 115, "max_ms": 289.517, "p95_ms": 259.61}, "triton_wait": {"avg_ms": 75.318, "count": 116, "max_ms": 121.493, "p95_ms": 103.479}, "visualize": {"avg_ms": 9.576, "count": 116, "max_ms": 12.702, "p95_ms": 10.985}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.864, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 77}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4179.906, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555d8c2042c0] error while decoding MB 74 20, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:25:24+09:00", "last_error_summary": "[h264 @ 0x555d8c2042c0] error while decoding MB 74 20, bytestream -7", "last_exit_at": "2026-07-01T15:23:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.313, "last_started_at": "2026-07-01T15:23:36+09:00", "pid": 4155, "process": {"exists": false, "pid": 4155}, "restart_count": 18, "restart_count_per_hour": 18, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.062, "sample_count": 1303, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9307, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:25:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T15:25:24+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T15:25:24+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-01T15:25:24.821462+09:00", "mtime_epoch": 1782887124.821462, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202872}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:25:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 859.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.609, "pushed": 4.609, "read": 4.609}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.464, "count": 137, "max_ms": 11.912, "p95_ms": 8.965}, "gstreamer_push": {"avg_ms": 0.938, "count": 138, "max_ms": 2.017, "p95_ms": 1.821}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.88, "count": 137, "max_ms": 84.391, "p95_ms": 68.481}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.047, "count": 137, "max_ms": 110.166, "p95_ms": 108.238}, "yolo11_640_fire_11class": {"avg_ms": 57.092, "count": 137, "max_ms": 90.322, "p95_ms": 79.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.089, "count": 137, "max_ms": 111.196, "p95_ms": 103.701}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.284, "count": 138, "max_ms": 274.543, "p95_ms": 246.465}, "read_interval": {"avg_ms": 218.088, "count": 138, "max_ms": 283.78, "p95_ms": 257.175}, "triton_wait": {"avg_ms": 75.55, "count": 137, "max_ms": 111.209, "p95_ms": 108.3}, "visualize": {"avg_ms": 9.307, "count": 137, "max_ms": 12.924, "p95_ms": 10.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 77}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4178.652, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:25:24+09:00", "last_error_summary": "[h264 @ 0x555d8c2042c0] error while decoding MB 74 20, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:25:24+09:00", "last_error_summary": "[h264 @ 0x555d8c2042c0] error while decoding MB 74 20, bytestream -7", "last_exit_at": "2026-07-01T15:25:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.45, "last_started_at": "2026-07-01T15:23:36+09:00", "pid": 4155, "process": {"exists": false, "pid": 4155}, "restart_count": 18, "restart_count_per_hour": 18, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.062, "sample_count": 1303, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9308, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:25:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:25:25+09:00", "last_frame_read_age_sec": 0.193, "last_frame_read_at": "2026-07-01T15:25:24+09:00", "mjpeg": {"age_sec": 0.19, "exists": true, "mtime": "2026-07-01T15:25:24.821462+09:00", "mtime_epoch": 1782887124.821462, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202872}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:25:25+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 859.07, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.628, "pushed": 4.628, "read": 4.594}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.481, "count": 138, "max_ms": 11.912, "p95_ms": 9.072}, "gstreamer_push": {"avg_ms": 0.938, "count": 138, "max_ms": 2.017, "p95_ms": 1.821}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.906, "count": 138, "max_ms": 84.391, "p95_ms": 68.481}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.992, "count": 138, "max_ms": 110.166, "p95_ms": 108.238}, "yolo11_640_fire_11class": {"avg_ms": 57.06, "count": 138, "max_ms": 90.322, "p95_ms": 79.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.029, "count": 138, "max_ms": 111.196, "p95_ms": 103.701}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.14, "count": 138, "max_ms": 274.543, "p95_ms": 246.465}, "read_interval": {"avg_ms": 218.091, "count": 137, "max_ms": 283.78, "p95_ms": 257.175}, "triton_wait": {"avg_ms": 75.47, "count": 138, "max_ms": 111.209, "p95_ms": 108.3}, "visualize": {"avg_ms": 9.316, "count": 138, "max_ms": 12.924, "p95_ms": 10.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.821, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 77}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4178.594, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:25:24+09:00", "last_error_summary": "[h264 @ 0x555d8c2042c0] error while decoding MB 74 20, bytestream -7", "last_exit_at": "2026-07-01T15:25:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.454, "last_started_at": "2026-07-01T15:23:36+09:00", "pid": 4155, "process": {"exists": false, "pid": 4155}, "restart_count": 18, "restart_count_per_hour": 18, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.062, "sample_count": 1303, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9308, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:25:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:25:25+09:00", "last_frame_read_age_sec": 0.196, "last_frame_read_at": "2026-07-01T15:25:24+09:00", "mjpeg": {"age_sec": 0.193, "exists": true, "mtime": "2026-07-01T15:25:24.821462+09:00", "mtime_epoch": 1782887124.821462, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202872}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:25:25+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 859.07, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.627, "pushed": 4.627, "read": 4.593}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.481, "count": 138, "max_ms": 11.912, "p95_ms": 9.072}, "gstreamer_push": {"avg_ms": 0.938, "count": 138, "max_ms": 2.017, "p95_ms": 1.821}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.906, "count": 138, "max_ms": 84.391, "p95_ms": 68.481}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.992, "count": 138, "max_ms": 110.166, "p95_ms": 108.238}, "yolo11_640_fire_11class": {"avg_ms": 57.06, "count": 138, "max_ms": 90.322, "p95_ms": 79.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.029, "count": 138, "max_ms": 111.196, "p95_ms": 103.701}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.14, "count": 138, "max_ms": 274.543, "p95_ms": 246.465}, "read_interval": {"avg_ms": 218.091, "count": 137, "max_ms": 283.78, "p95_ms": 257.175}, "triton_wait": {"avg_ms": 75.47, "count": 138, "max_ms": 111.209, "p95_ms": 108.3}, "visualize": {"avg_ms": 9.316, "count": 138, "max_ms": 12.924, "p95_ms": 10.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.825, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 77}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4282.844, "limit_mb": 8796093022207.996}, "detail": {"pid": 4262, "restart_count": 19, "start_count": 20}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:25:24+09:00", "last_error_summary": "[h264 @ 0x555d8c2042c0] error while decoding MB 74 20, bytestream -7", "last_exit_at": "2026-07-01T15:25:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:25:27+09:00", "pid": 4262, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.117, "name": "ffmpeg", "pid": 4262, "rss_mb": 137.117, "state": "S (sleeping)", "threads": 106}, "restart_count": 19, "restart_count_per_hour": 19, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.062, "sample_count": 1281, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9308, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:25:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.805, "last_frame_pushed_at": "2026-07-01T15:25:25+09:00", "last_frame_read_age_sec": 4.995, "last_frame_read_at": "2026-07-01T15:25:24+09:00", "mjpeg": {"age_sec": 4.792, "exists": true, "mtime": "2026-07-01T15:25:25.021465+09:00", "mtime_epoch": 1782887125.021465, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203259}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:25:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 859.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.9, "pushed": 3.9, "read": 3.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.388, "count": 116, "max_ms": 9.839, "p95_ms": 8.919}, "gstreamer_push": {"avg_ms": 0.919, "count": 117, "max_ms": 1.933, "p95_ms": 1.103}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.662, "count": 116, "max_ms": 84.391, "p95_ms": 68.481}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.143, "count": 116, "max_ms": 110.166, "p95_ms": 106.053}, "yolo11_640_fire_11class": {"avg_ms": 57.617, "count": 116, "max_ms": 90.322, "p95_ms": 79.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.825, "count": 116, "max_ms": 111.196, "p95_ms": 103.914}}, "truncated_count": 0}, "processing_total": {"avg_ms": 207.286, "count": 117, "max_ms": 274.543, "p95_ms": 246.465}, "read_interval": {"avg_ms": 217.207, "count": 116, "max_ms": 283.78, "p95_ms": 257.175}, "triton_wait": {"avg_ms": 75.784, "count": 116, "max_ms": 111.209, "p95_ms": 108.251}, "visualize": {"avg_ms": 9.315, "count": 116, "max_ms": 12.924, "p95_ms": 10.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_152540_945a26ba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 4288.316, "limit_mb": 8796093022207.996}, "detail": {"pid": 4371, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:25:40+09:00", "pid": 4371, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.535, "name": "ffmpeg", "pid": 4371, "rss_mb": 137.535, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.788}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "945a26ba", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 3.192, "exists": true, "mtime": "2026-07-01T15:25:39.717689+09:00", "mtime_epoch": 1782887139.7176893, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 255989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T15:25:42+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 709.395, "name": "celery", "pid": 239, "rss_mb": 679.477, "state": "R (running)", "threads": 118}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T15:25:40+09:00", "state": "running", "task_id": "8a6c06b9-d38f-4130-b8c8-c749e2e2e721", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_152542_c0873c01.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 4298.012, "limit_mb": 8796093022207.996}, "detail": {"pid": 4426, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:25:42+09:00", "pid": 4426, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.859, "name": "ffmpeg", "pid": 4426, "rss_mb": 137.859, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.733}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c0873c01", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 4.392, "exists": true, "mtime": "2026-07-01T15:25:39.717689+09:00", "mtime_epoch": 1782887139.7176893, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 255989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T15:25:44+09:00", "process": {"exists": true, "fd_count": 111, "hwm_mb": 470.695, "name": "celery", "pid": 241, "rss_mb": 470.695, "state": "R (running)", "threads": 7}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T15:25:41+09:00", "state": "running", "task_id": "8114cbfb-d26e-487f-adb8-77b0d9aba2dc", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_152543_c8f6f07c.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 4297.652, "limit_mb": 8796093022207.996}, "detail": {"pid": 4554, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:25:43+09:00", "pid": 4554, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.523, "name": "ffmpeg", "pid": 4554, "rss_mb": 137.523, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.735}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "c8f6f07c", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 5.483, "exists": true, "mtime": "2026-07-01T15:25:39.717689+09:00", "mtime_epoch": 1782887139.7176893, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 255989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T15:25:45+09:00", "process": {"exists": true, "fd_count": 112, "hwm_mb": 470.699, "name": "celery", "pid": 243, "rss_mb": 470.699, "state": "R (running)", "threads": 7}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T15:25:42+09:00", "state": "running", "task_id": "2a42b316-0135-43b7-b2ea-be4cc3711448", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260701_152544_78e66279.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 4226.75, "limit_mb": 8796093022207.996}, "detail": {"pid": 4709, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:25:44+09:00", "pid": 4709, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.668, "name": "ffmpeg", "pid": 4709, "rss_mb": 137.668, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.805}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "ok", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "78e66279", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 6.54, "exists": true, "mtime": "2026-07-01T15:25:39.717689+09:00", "mtime_epoch": 1782887139.7176893, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 255989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-01T15:25:46+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 709.395, "name": "celery", "pid": 239, "rss_mb": 679.477, "state": "R (running)", "threads": 118}, "reason_codes": [], "recommended_action": "observe", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T15:25:44+09:00", "state": "running", "task_id": "fcb78d34-3d0e-46ba-8f6c-b1e6532003e0", "trigger": "update"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 82}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4162.996, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c92fa85680] error while decoding MB 78 29, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:27:38+09:00", "last_error_summary": "[h264 @ 0x55c92fa85680] error while decoding MB 78 29, bytestream -9", "last_exit_at": "2026-07-01T15:25:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.41, "last_started_at": "2026-07-01T15:25:27+09:00", "pid": 4262, "process": {"exists": false, "pid": 4262}, "restart_count": 19, "restart_count_per_hour": 19, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.016, "sample_count": 1298, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9881, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:27:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T15:27:38+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T15:27:38+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T15:27:38.179506+09:00", "mtime_epoch": 1782887258.1795058, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205186}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:27:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.973, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.449, "pushed": 4.449, "read": 4.449}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.939, "count": 133, "max_ms": 12.386, "p95_ms": 9.878}, "gstreamer_push": {"avg_ms": 0.958, "count": 133, "max_ms": 3.093, "p95_ms": 1.853}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.526, "count": 133, "max_ms": 80.034, "p95_ms": 67.547}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.807, "count": 133, "max_ms": 114.197, "p95_ms": 103.671}, "yolo11_640_fire_11class": {"avg_ms": 56.739, "count": 133, "max_ms": 86.085, "p95_ms": 76.447}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.073, "count": 133, "max_ms": 107.705, "p95_ms": 98.183}}, "truncated_count": 0}, "processing_total": {"avg_ms": 216.499, "count": 133, "max_ms": 282.856, "p95_ms": 248.721}, "read_interval": {"avg_ms": 225.986, "count": 133, "max_ms": 293.079, "p95_ms": 257.69}, "triton_wait": {"avg_ms": 77.245, "count": 133, "max_ms": 114.21, "p95_ms": 103.799}, "visualize": {"avg_ms": 9.706, "count": 133, "max_ms": 14.165, "p95_ms": 12.349}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 82}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4162.078, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:27:38+09:00", "last_error_summary": "[h264 @ 0x55c92fa85680] error while decoding MB 78 29, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:27:38+09:00", "last_error_summary": "[h264 @ 0x55c92fa85680] error while decoding MB 78 29, bytestream -9", "last_exit_at": "2026-07-01T15:27:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.597, "last_started_at": "2026-07-01T15:25:27+09:00", "pid": 4262, "process": {"exists": false, "pid": 4262}, "restart_count": 19, "restart_count_per_hour": 19, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.016, "sample_count": 1299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:27:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:27:38+09:00", "last_frame_read_age_sec": 0.241, "last_frame_read_at": "2026-07-01T15:27:38+09:00", "mjpeg": {"age_sec": 0.23, "exists": true, "mtime": "2026-07-01T15:27:38.179506+09:00", "mtime_epoch": 1782887258.1795058, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205186}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:27:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.973, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.455, "pushed": 4.455, "read": 4.421}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.946, "count": 133, "max_ms": 12.386, "p95_ms": 9.878}, "gstreamer_push": {"avg_ms": 0.96, "count": 133, "max_ms": 3.093, "p95_ms": 1.853}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.474, "count": 133, "max_ms": 80.034, "p95_ms": 67.547}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.773, "count": 133, "max_ms": 114.197, "p95_ms": 103.671}, "yolo11_640_fire_11class": {"avg_ms": 56.634, "count": 133, "max_ms": 86.085, "p95_ms": 74.541}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.908, "count": 133, "max_ms": 107.705, "p95_ms": 95.019}}, "truncated_count": 0}, "processing_total": {"avg_ms": 216.621, "count": 133, "max_ms": 282.856, "p95_ms": 248.721}, "read_interval": {"avg_ms": 225.957, "count": 132, "max_ms": 293.079, "p95_ms": 257.69}, "triton_wait": {"avg_ms": 77.201, "count": 133, "max_ms": 114.21, "p95_ms": 103.799}, "visualize": {"avg_ms": 9.72, "count": 133, "max_ms": 14.165, "p95_ms": 12.349}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.855, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 82}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4162.07, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:27:38+09:00", "last_error_summary": "[h264 @ 0x55c92fa85680] error while decoding MB 78 29, bytestream -9", "last_exit_at": "2026-07-01T15:27:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.601, "last_started_at": "2026-07-01T15:25:27+09:00", "pid": 4262, "process": {"exists": false, "pid": 4262}, "restart_count": 19, "restart_count_per_hour": 19, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.016, "sample_count": 1299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:27:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T15:27:38+09:00", "last_frame_read_age_sec": 0.245, "last_frame_read_at": "2026-07-01T15:27:38+09:00", "mjpeg": {"age_sec": 0.234, "exists": true, "mtime": "2026-07-01T15:27:38.179506+09:00", "mtime_epoch": 1782887258.1795058, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205186}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:27:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.973, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.454, "pushed": 4.454, "read": 4.421}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.946, "count": 133, "max_ms": 12.386, "p95_ms": 9.878}, "gstreamer_push": {"avg_ms": 0.96, "count": 133, "max_ms": 3.093, "p95_ms": 1.853}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.474, "count": 133, "max_ms": 80.034, "p95_ms": 67.547}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.773, "count": 133, "max_ms": 114.197, "p95_ms": 103.671}, "yolo11_640_fire_11class": {"avg_ms": 56.634, "count": 133, "max_ms": 86.085, "p95_ms": 74.541}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.908, "count": 133, "max_ms": 107.705, "p95_ms": 95.019}}, "truncated_count": 0}, "processing_total": {"avg_ms": 216.621, "count": 133, "max_ms": 282.856, "p95_ms": 248.721}, "read_interval": {"avg_ms": 225.957, "count": 132, "max_ms": 293.079, "p95_ms": 257.69}, "triton_wait": {"avg_ms": 77.201, "count": 133, "max_ms": 114.21, "p95_ms": 103.799}, "visualize": {"avg_ms": 9.72, "count": 133, "max_ms": 14.165, "p95_ms": 12.349}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.859, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 82}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4265.301, "limit_mb": 8796093022207.996}, "detail": {"pid": 4995, "restart_count": 20, "start_count": 21}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:27:38+09:00", "last_error_summary": "[h264 @ 0x55c92fa85680] error while decoding MB 78 29, bytestream -9", "last_exit_at": "2026-07-01T15:27:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:27:40+09:00", "pid": 4995, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.781, "name": "ffmpeg", "pid": 4995, "rss_mb": 141.781, "state": "S (sleeping)", "threads": 106}, "restart_count": 20, "restart_count_per_hour": 20, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 251.79, "sample_count": 1280, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 9882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:27:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.416, "last_frame_pushed_at": "2026-07-01T15:27:38+09:00", "last_frame_read_age_sec": 4.653, "last_frame_read_at": "2026-07-01T15:27:38+09:00", "mjpeg": {"age_sec": 4.394, "exists": true, "mtime": "2026-07-01T15:27:38.427510+09:00", "mtime_epoch": 1782887258.4275098, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205490}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:27:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 866.078, "name": "celery", "pid": 237, "rss_mb": 861.973, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.767, "pushed": 3.8, "read": 3.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.991, "count": 113, "max_ms": 12.386, "p95_ms": 9.878}, "gstreamer_push": {"avg_ms": 0.966, "count": 114, "max_ms": 3.093, "p95_ms": 1.931}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.152, "count": 113, "max_ms": 80.034, "p95_ms": 67.149}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.078, "count": 113, "max_ms": 114.197, "p95_ms": 103.671}, "yolo11_640_fire_11class": {"avg_ms": 56.759, "count": 113, "max_ms": 86.085, "p95_ms": 77.277}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.914, "count": 113, "max_ms": 107.705, "p95_ms": 95.019}}, "truncated_count": 0}, "processing_total": {"avg_ms": 216.899, "count": 113, "max_ms": 282.856, "p95_ms": 248.721}, "read_interval": {"avg_ms": 226.144, "count": 113, "max_ms": 293.079, "p95_ms": 257.69}, "triton_wait": {"avg_ms": 77.35, "count": 113, "max_ms": 114.21, "p95_ms": 103.799}, "visualize": {"avg_ms": 9.735, "count": 113, "max_ms": 14.165, "p95_ms": 12.349}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 87}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3908.07, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:27:38+09:00", "last_error_summary": "[h264 @ 0x55c92fa85680] error while decoding MB 78 29, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:27:38+09:00", "last_error_summary": "[h264 @ 0x55c92fa85680] error while decoding MB 78 29, bytestream -9", "last_exit_at": "2026-07-01T15:29:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.037, "last_started_at": "2026-07-01T15:27:40+09:00", "pid": 4995, "process": {"exists": false, "pid": 4995}, "restart_count": 20, "restart_count_per_hour": 20, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 247.842, "sample_count": 1335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10477, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:29:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:29:48+09:00", "last_frame_read_age_sec": 0.192, "last_frame_read_at": "2026-07-01T15:29:48+09:00", "mjpeg": {"age_sec": 0.179, "exists": true, "mtime": "2026-07-01T15:29:48.677906+09:00", "mtime_epoch": 1782887388.677906, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206610}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:29:48+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 868.344, "name": "celery", "pid": 237, "rss_mb": 860.316, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.888, "pushed": 4.888, "read": 4.854}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.436, "count": 146, "max_ms": 12.603, "p95_ms": 8.696}, "gstreamer_push": {"avg_ms": 0.927, "count": 146, "max_ms": 2.919, "p95_ms": 1.037}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.822, "count": 146, "max_ms": 71.412, "p95_ms": 62.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.354, "count": 146, "max_ms": 92.907, "p95_ms": 88.328}, "yolo11_640_fire_11class": {"avg_ms": 53.363, "count": 146, "max_ms": 78.842, "p95_ms": 71.114}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.962, "count": 146, "max_ms": 92.853, "p95_ms": 87.089}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.485, "count": 146, "max_ms": 287.867, "p95_ms": 223.926}, "read_interval": {"avg_ms": 206.202, "count": 145, "max_ms": 294.602, "p95_ms": 233.756}, "triton_wait": {"avg_ms": 66.522, "count": 146, "max_ms": 92.919, "p95_ms": 88.823}, "visualize": {"avg_ms": 9.22, "count": 146, "max_ms": 13.539, "p95_ms": 10.129}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 87}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3908.062, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:27:38+09:00", "last_error_summary": "[h264 @ 0x55c92fa85680] error while decoding MB 78 29, bytestream -9", "last_exit_at": "2026-07-01T15:29:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.04, "last_started_at": "2026-07-01T15:27:40+09:00", "pid": 4995, "process": {"exists": false, "pid": 4995}, "restart_count": 20, "restart_count_per_hour": 20, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 247.842, "sample_count": 1335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10477, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:29:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:29:48+09:00", "last_frame_read_age_sec": 0.195, "last_frame_read_at": "2026-07-01T15:29:48+09:00", "mjpeg": {"age_sec": 0.183, "exists": true, "mtime": "2026-07-01T15:29:48.677906+09:00", "mtime_epoch": 1782887388.677906, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206610}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:29:48+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 868.344, "name": "celery", "pid": 237, "rss_mb": 860.316, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.887, "pushed": 4.887, "read": 4.854}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.436, "count": 146, "max_ms": 12.603, "p95_ms": 8.696}, "gstreamer_push": {"avg_ms": 0.927, "count": 146, "max_ms": 2.919, "p95_ms": 1.037}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.822, "count": 146, "max_ms": 71.412, "p95_ms": 62.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.354, "count": 146, "max_ms": 92.907, "p95_ms": 88.328}, "yolo11_640_fire_11class": {"avg_ms": 53.363, "count": 146, "max_ms": 78.842, "p95_ms": 71.114}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.962, "count": 146, "max_ms": 92.853, "p95_ms": 87.089}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.485, "count": 146, "max_ms": 287.867, "p95_ms": 223.926}, "read_interval": {"avg_ms": 206.202, "count": 145, "max_ms": 294.602, "p95_ms": 233.756}, "triton_wait": {"avg_ms": 66.522, "count": 146, "max_ms": 92.919, "p95_ms": 88.823}, "visualize": {"avg_ms": 9.22, "count": 146, "max_ms": 13.539, "p95_ms": 10.129}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 87}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4013.156, "limit_mb": 8796093022207.996}, "detail": {"pid": 5102, "restart_count": 21, "start_count": 22}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:27:38+09:00", "last_error_summary": "[h264 @ 0x55c92fa85680] error while decoding MB 78 29, bytestream -9", "last_exit_at": "2026-07-01T15:29:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:29:50+09:00", "pid": 5102, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.934, "name": "ffmpeg", "pid": 5102, "rss_mb": 138.934, "state": "S (sleeping)", "threads": 106}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 247.93, "sample_count": 1316, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10477, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:29:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.139, "last_frame_pushed_at": "2026-07-01T15:29:48+09:00", "last_frame_read_age_sec": 4.327, "last_frame_read_at": "2026-07-01T15:29:48+09:00", "mjpeg": {"age_sec": 4.123, "exists": true, "mtime": "2026-07-01T15:29:48.869911+09:00", "mtime_epoch": 1782887388.8699114, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206302}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:29:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 868.344, "name": "celery", "pid": 237, "rss_mb": 860.316, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.167, "pushed": 4.2, "read": 4.167}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.273, "count": 125, "max_ms": 9.778, "p95_ms": 8.084}, "gstreamer_push": {"avg_ms": 0.933, "count": 126, "max_ms": 2.919, "p95_ms": 0.969}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.006, "count": 125, "max_ms": 71.412, "p95_ms": 62.461}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.598, "count": 125, "max_ms": 92.907, "p95_ms": 88.328}, "yolo11_640_fire_11class": {"avg_ms": 54.56, "count": 125, "max_ms": 78.842, "p95_ms": 71.61}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.158, "count": 125, "max_ms": 92.853, "p95_ms": 87.51}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.177, "count": 125, "max_ms": 287.867, "p95_ms": 224.643}, "read_interval": {"avg_ms": 206.873, "count": 125, "max_ms": 294.602, "p95_ms": 234.849}, "triton_wait": {"avg_ms": 67.735, "count": 125, "max_ms": 92.919, "p95_ms": 89.226}, "visualize": {"avg_ms": 9.184, "count": 125, "max_ms": 12.949, "p95_ms": 10.046}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 92}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3913.047, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55bdda229340] error while decoding MB 25 32, bytestream -46", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:31:59+09:00", "last_error_summary": "[h264 @ 0x55bdda229340] error while decoding MB 25 32, bytestream -46", "last_exit_at": "2026-07-01T15:29:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.993, "last_started_at": "2026-07-01T15:29:50+09:00", "pid": 5102, "process": {"exists": false, "pid": 5102}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.35, "sample_count": 1384, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11093, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:31:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.048, "last_frame_pushed_at": "2026-07-01T15:31:59+09:00", "last_frame_read_age_sec": 0.039, "last_frame_read_at": "2026-07-01T15:31:59+09:00", "mjpeg": {"age_sec": 0.031, "exists": true, "mtime": "2026-07-01T15:31:59.953379+09:00", "mtime_epoch": 1782887519.953379, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206347}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:31:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 868.938, "name": "celery", "pid": 237, "rss_mb": 865.953, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.986, "pushed": 4.986, "read": 4.986}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.166, "count": 149, "max_ms": 9.271, "p95_ms": 8.024}, "gstreamer_push": {"avg_ms": 0.92, "count": 149, "max_ms": 2.806, "p95_ms": 1.125}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.794, "count": 149, "max_ms": 71.599, "p95_ms": 68.155}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.528, "count": 149, "max_ms": 95.826, "p95_ms": 87.24}, "yolo11_640_fire_11class": {"avg_ms": 53.1, "count": 149, "max_ms": 76.259, "p95_ms": 70.366}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.269, "count": 149, "max_ms": 98.367, "p95_ms": 88.744}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.157, "count": 149, "max_ms": 245.779, "p95_ms": 221.69}, "read_interval": {"avg_ms": 201.657, "count": 149, "max_ms": 256.219, "p95_ms": 232.33}, "triton_wait": {"avg_ms": 66.638, "count": 149, "max_ms": 98.379, "p95_ms": 89.567}, "visualize": {"avg_ms": 8.819, "count": 149, "max_ms": 11.576, "p95_ms": 9.969}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 92}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3912.34, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:31:59+09:00", "last_error_summary": "[h264 @ 0x55bdda229340] error while decoding MB 25 32, bytestream -46", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:31:59+09:00", "last_error_summary": "[h264 @ 0x55bdda229340] error while decoding MB 25 32, bytestream -46", "last_exit_at": "2026-07-01T15:32:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.144, "last_started_at": "2026-07-01T15:29:50+09:00", "pid": 5102, "process": {"exists": false, "pid": 5102}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.35, "sample_count": 1384, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11094, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:31:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:32:00+09:00", "last_frame_read_age_sec": 0.19, "last_frame_read_at": "2026-07-01T15:31:59+09:00", "mjpeg": {"age_sec": 0.182, "exists": true, "mtime": "2026-07-01T15:31:59.953379+09:00", "mtime_epoch": 1782887519.953379, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206347}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:32:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 868.938, "name": "celery", "pid": 237, "rss_mb": 865.953, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.994, "pushed": 4.994, "read": 4.96}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.175, "count": 149, "max_ms": 9.271, "p95_ms": 8.032}, "gstreamer_push": {"avg_ms": 0.92, "count": 149, "max_ms": 2.806, "p95_ms": 1.125}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.83, "count": 149, "max_ms": 71.599, "p95_ms": 68.155}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.372, "count": 149, "max_ms": 95.826, "p95_ms": 87.24}, "yolo11_640_fire_11class": {"avg_ms": 52.99, "count": 149, "max_ms": 76.259, "p95_ms": 70.366}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.227, "count": 149, "max_ms": 98.367, "p95_ms": 88.744}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.03, "count": 149, "max_ms": 245.779, "p95_ms": 221.69}, "read_interval": {"avg_ms": 201.568, "count": 148, "max_ms": 256.219, "p95_ms": 232.33}, "triton_wait": {"avg_ms": 66.504, "count": 149, "max_ms": 98.379, "p95_ms": 89.567}, "visualize": {"avg_ms": 8.82, "count": 149, "max_ms": 11.576, "p95_ms": 9.969}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.836, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 92}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3912.34, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:31:59+09:00", "last_error_summary": "[h264 @ 0x55bdda229340] error while decoding MB 25 32, bytestream -46", "last_exit_at": "2026-07-01T15:32:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.147, "last_started_at": "2026-07-01T15:29:50+09:00", "pid": 5102, "process": {"exists": false, "pid": 5102}, "restart_count": 21, "restart_count_per_hour": 21, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.35, "sample_count": 1384, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11094, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:31:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:32:00+09:00", "last_frame_read_age_sec": 0.193, "last_frame_read_at": "2026-07-01T15:31:59+09:00", "mjpeg": {"age_sec": 0.186, "exists": true, "mtime": "2026-07-01T15:31:59.953379+09:00", "mtime_epoch": 1782887519.953379, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206347}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:32:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 868.938, "name": "celery", "pid": 237, "rss_mb": 865.953, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.993, "pushed": 4.993, "read": 4.96}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.175, "count": 149, "max_ms": 9.271, "p95_ms": 8.032}, "gstreamer_push": {"avg_ms": 0.92, "count": 149, "max_ms": 2.806, "p95_ms": 1.125}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.83, "count": 149, "max_ms": 71.599, "p95_ms": 68.155}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.372, "count": 149, "max_ms": 95.826, "p95_ms": 87.24}, "yolo11_640_fire_11class": {"avg_ms": 52.99, "count": 149, "max_ms": 76.259, "p95_ms": 70.366}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.227, "count": 149, "max_ms": 98.367, "p95_ms": 88.744}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.03, "count": 149, "max_ms": 245.779, "p95_ms": 221.69}, "read_interval": {"avg_ms": 201.568, "count": 148, "max_ms": 256.219, "p95_ms": 232.33}, "triton_wait": {"avg_ms": 66.504, "count": 149, "max_ms": 98.379, "p95_ms": 89.567}, "visualize": {"avg_ms": 8.82, "count": 149, "max_ms": 11.576, "p95_ms": 9.969}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.84, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 92}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4014.461, "limit_mb": 8796093022207.996}, "detail": {"pid": 5209, "restart_count": 22, "start_count": 23}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:31:59+09:00", "last_error_summary": "[h264 @ 0x55bdda229340] error while decoding MB 25 32, bytestream -46", "last_exit_at": "2026-07-01T15:32:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:32:02+09:00", "pid": 5209, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.355, "name": "ffmpeg", "pid": 5209, "rss_mb": 136.355, "state": "S (sleeping)", "threads": 106}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 238.35, "sample_count": 1362, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11094, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:31:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.671, "last_frame_pushed_at": "2026-07-01T15:32:00+09:00", "last_frame_read_age_sec": 4.858, "last_frame_read_at": "2026-07-01T15:31:59+09:00", "mjpeg": {"age_sec": 4.655, "exists": true, "mtime": "2026-07-01T15:32:00.149384+09:00", "mtime_epoch": 1782887520.1493838, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206390}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:32:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 868.938, "name": "celery", "pid": 237, "rss_mb": 865.953, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.209, "pushed": 4.209, "read": 4.176}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.236, "count": 125, "max_ms": 9.271, "p95_ms": 8.032}, "gstreamer_push": {"avg_ms": 0.898, "count": 126, "max_ms": 1.9, "p95_ms": 1.003}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.242, "count": 125, "max_ms": 71.599, "p95_ms": 68.29}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.81, "count": 125, "max_ms": 95.826, "p95_ms": 87.81}, "yolo11_640_fire_11class": {"avg_ms": 53.369, "count": 125, "max_ms": 76.259, "p95_ms": 70.805}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.836, "count": 125, "max_ms": 98.367, "p95_ms": 89.33}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.954, "count": 126, "max_ms": 245.779, "p95_ms": 225.094}, "read_interval": {"avg_ms": 202.434, "count": 125, "max_ms": 256.219, "p95_ms": 234.63}, "triton_wait": {"avg_ms": 67.085, "count": 125, "max_ms": 98.379, "p95_ms": 89.686}, "visualize": {"avg_ms": 8.813, "count": 125, "max_ms": 10.796, "p95_ms": 9.944}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 97}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3911.898, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563f999e0c00] error while decoding MB 30 7, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:34:11+09:00", "last_error_summary": "[h264 @ 0x563f999e0c00] error while decoding MB 30 7, bytestream -5", "last_exit_at": "2026-07-01T15:32:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.25, "last_started_at": "2026-07-01T15:32:02+09:00", "pid": 5209, "process": {"exists": false, "pid": 5209}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.389, "sample_count": 1403, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:34:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T15:34:10+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T15:34:11+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-01T15:34:10.996572+09:00", "mtime_epoch": 1782887650.9965718, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203745}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:34:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 871.957, "name": "celery", "pid": 237, "rss_mb": 869.379, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 143, "read_frames": 143, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.733, "pushed": 4.767, "read": 4.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.474, "count": 142, "max_ms": 11.094, "p95_ms": 8.852}, "gstreamer_push": {"avg_ms": 0.923, "count": 143, "max_ms": 2.884, "p95_ms": 1.259}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.826, "count": 142, "max_ms": 74.836, "p95_ms": 65.892}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.603, "count": 142, "max_ms": 95.114, "p95_ms": 87.561}, "yolo11_640_fire_11class": {"avg_ms": 54.569, "count": 142, "max_ms": 76.011, "p95_ms": 69.804}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.299, "count": 142, "max_ms": 92.775, "p95_ms": 89.58}}, "truncated_count": 0}, "processing_total": {"avg_ms": 199.587, "count": 142, "max_ms": 264.628, "p95_ms": 233.81}, "read_interval": {"avg_ms": 210.985, "count": 143, "max_ms": 446.061, "p95_ms": 243.678}, "triton_wait": {"avg_ms": 68.61, "count": 142, "max_ms": 95.127, "p95_ms": 89.592}, "visualize": {"avg_ms": 9.088, "count": 142, "max_ms": 12.084, "p95_ms": 10.383}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 97}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3911.141, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:34:11+09:00", "last_error_summary": "[h264 @ 0x563f999e0c00] error while decoding MB 30 7, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:34:11+09:00", "last_error_summary": "[h264 @ 0x563f999e0c00] error while decoding MB 30 7, bytestream -5", "last_exit_at": "2026-07-01T15:34:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.389, "last_started_at": "2026-07-01T15:32:02+09:00", "pid": 5209, "process": {"exists": false, "pid": 5209}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.389, "sample_count": 1403, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:34:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:34:11+09:00", "last_frame_read_age_sec": 0.191, "last_frame_read_at": "2026-07-01T15:34:11+09:00", "mjpeg": {"age_sec": 0.196, "exists": true, "mtime": "2026-07-01T15:34:10.996572+09:00", "mtime_epoch": 1782887650.9965718, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203745}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:34:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 871.957, "name": "celery", "pid": 237, "rss_mb": 869.379, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.778, "pushed": 4.778, "read": 4.745}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.462, "count": 142, "max_ms": 11.094, "p95_ms": 8.852}, "gstreamer_push": {"avg_ms": 0.923, "count": 143, "max_ms": 2.884, "p95_ms": 1.259}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.867, "count": 142, "max_ms": 74.836, "p95_ms": 65.892}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.606, "count": 142, "max_ms": 95.114, "p95_ms": 87.561}, "yolo11_640_fire_11class": {"avg_ms": 54.591, "count": 142, "max_ms": 76.011, "p95_ms": 69.804}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.291, "count": 142, "max_ms": 92.775, "p95_ms": 89.58}}, "truncated_count": 0}, "processing_total": {"avg_ms": 199.499, "count": 143, "max_ms": 264.628, "p95_ms": 233.81}, "read_interval": {"avg_ms": 210.832, "count": 142, "max_ms": 446.061, "p95_ms": 243.678}, "triton_wait": {"avg_ms": 68.612, "count": 142, "max_ms": 95.127, "p95_ms": 89.592}, "visualize": {"avg_ms": 9.07, "count": 142, "max_ms": 12.084, "p95_ms": 10.206}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 97}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3911.133, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:34:11+09:00", "last_error_summary": "[h264 @ 0x563f999e0c00] error while decoding MB 30 7, bytestream -5", "last_exit_at": "2026-07-01T15:34:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.393, "last_started_at": "2026-07-01T15:32:02+09:00", "pid": 5209, "process": {"exists": false, "pid": 5209}, "restart_count": 22, "restart_count_per_hour": 22, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.389, "sample_count": 1403, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:34:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:34:11+09:00", "last_frame_read_age_sec": 0.195, "last_frame_read_at": "2026-07-01T15:34:11+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T15:34:10.996572+09:00", "mtime_epoch": 1782887650.9965718, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203745}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:34:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 871.957, "name": "celery", "pid": 237, "rss_mb": 869.379, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.778, "pushed": 4.778, "read": 4.745}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.462, "count": 142, "max_ms": 11.094, "p95_ms": 8.852}, "gstreamer_push": {"avg_ms": 0.923, "count": 143, "max_ms": 2.884, "p95_ms": 1.259}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.867, "count": 142, "max_ms": 74.836, "p95_ms": 65.892}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.606, "count": 142, "max_ms": 95.114, "p95_ms": 87.561}, "yolo11_640_fire_11class": {"avg_ms": 54.591, "count": 142, "max_ms": 76.011, "p95_ms": 69.804}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.291, "count": 142, "max_ms": 92.775, "p95_ms": 89.58}}, "truncated_count": 0}, "processing_total": {"avg_ms": 199.499, "count": 143, "max_ms": 264.628, "p95_ms": 233.81}, "read_interval": {"avg_ms": 210.832, "count": 142, "max_ms": 446.061, "p95_ms": 243.678}, "triton_wait": {"avg_ms": 68.612, "count": 142, "max_ms": 95.127, "p95_ms": 89.592}, "visualize": {"avg_ms": 9.07, "count": 142, "max_ms": 12.084, "p95_ms": 10.206}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 97}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4021.797, "limit_mb": 8796093022207.996}, "detail": {"pid": 5316, "restart_count": 23, "start_count": 24}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:34:11+09:00", "last_error_summary": "[h264 @ 0x563f999e0c00] error while decoding MB 30 7, bytestream -5", "last_exit_at": "2026-07-01T15:34:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:34:14+09:00", "pid": 5316, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.059, "name": "ffmpeg", "pid": 5316, "rss_mb": 139.059, "state": "S (sleeping)", "threads": 106}, "restart_count": 23, "restart_count_per_hour": 23, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.389, "sample_count": 1379, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 11697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:34:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.986, "last_frame_pushed_at": "2026-07-01T15:34:11+09:00", "last_frame_read_age_sec": 5.174, "last_frame_read_at": "2026-07-01T15:34:11+09:00", "mjpeg": {"age_sec": 4.979, "exists": true, "mtime": "2026-07-01T15:34:11.196577+09:00", "mtime_epoch": 1782887651.1965766, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194513}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:34:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 871.957, "name": "celery", "pid": 237, "rss_mb": 869.379, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.972, "pushed": 3.972, "read": 3.938}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.525, "count": 118, "max_ms": 11.094, "p95_ms": 9.052}, "gstreamer_push": {"avg_ms": 0.938, "count": 119, "max_ms": 2.884, "p95_ms": 1.771}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.808, "count": 118, "max_ms": 74.836, "p95_ms": 67.319}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.173, "count": 118, "max_ms": 95.114, "p95_ms": 87.561}, "yolo11_640_fire_11class": {"avg_ms": 54.467, "count": 118, "max_ms": 76.011, "p95_ms": 69.804}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.139, "count": 118, "max_ms": 92.775, "p95_ms": 89.58}}, "truncated_count": 0}, "processing_total": {"avg_ms": 200.08, "count": 119, "max_ms": 264.628, "p95_ms": 234.279}, "read_interval": {"avg_ms": 211.752, "count": 118, "max_ms": 446.061, "p95_ms": 244.929}, "triton_wait": {"avg_ms": 68.475, "count": 118, "max_ms": 95.127, "p95_ms": 89.592}, "visualize": {"avg_ms": 9.112, "count": 118, "max_ms": 12.084, "p95_ms": 10.667}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3912.668, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5623024f5900] error while decoding MB 46 19, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:36:19+09:00", "last_error_summary": "[h264 @ 0x5623024f5900] error while decoding MB 46 19, bytestream -9", "last_exit_at": "2026-07-01T15:34:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.727, "last_started_at": "2026-07-01T15:34:14+09:00", "pid": 5316, "process": {"exists": false, "pid": 5316}, "restart_count": 23, "restart_count_per_hour": 23, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.389, "sample_count": 1398, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12291, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:36:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T15:36:19+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T15:36:19+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T15:36:19.863505+09:00", "mtime_epoch": 1782887779.8635051, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199834}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:36:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 866.574, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.87, "pushed": 4.87, "read": 4.87}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.385, "count": 145, "max_ms": 10.925, "p95_ms": 7.87}, "gstreamer_push": {"avg_ms": 0.924, "count": 146, "max_ms": 3.055, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.855, "count": 145, "max_ms": 69.864, "p95_ms": 62.664}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.123, "count": 145, "max_ms": 94.171, "p95_ms": 86.148}, "yolo11_640_fire_11class": {"avg_ms": 53.374, "count": 145, "max_ms": 71.209, "p95_ms": 69.532}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.614, "count": 145, "max_ms": 92.404, "p95_ms": 88.403}}, "truncated_count": 0}, "processing_total": {"avg_ms": 196.528, "count": 146, "max_ms": 274.794, "p95_ms": 228.396}, "read_interval": {"avg_ms": 206.271, "count": 146, "max_ms": 286.265, "p95_ms": 238.638}, "triton_wait": {"avg_ms": 67.152, "count": 145, "max_ms": 94.183, "p95_ms": 89.129}, "visualize": {"avg_ms": 8.989, "count": 145, "max_ms": 12.097, "p95_ms": 10.279}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3912.027, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:36:19+09:00", "last_error_summary": "[h264 @ 0x5623024f5900] error while decoding MB 46 19, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:36:19+09:00", "last_error_summary": "[h264 @ 0x5623024f5900] error while decoding MB 46 19, bytestream -9", "last_exit_at": "2026-07-01T15:36:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.908, "last_started_at": "2026-07-01T15:34:14+09:00", "pid": 5316, "process": {"exists": false, "pid": 5316}, "restart_count": 23, "restart_count_per_hour": 23, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.389, "sample_count": 1398, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:36:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:36:20+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T15:36:19+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-01T15:36:19.863505+09:00", "mtime_epoch": 1782887779.8635051, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199834}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:36:20+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 866.574, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.872, "pushed": 4.872, "read": 4.839}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.392, "count": 145, "max_ms": 10.925, "p95_ms": 7.87}, "gstreamer_push": {"avg_ms": 0.911, "count": 146, "max_ms": 3.055, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.936, "count": 145, "max_ms": 69.864, "p95_ms": 62.664}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.264, "count": 145, "max_ms": 94.171, "p95_ms": 86.148}, "yolo11_640_fire_11class": {"avg_ms": 53.448, "count": 145, "max_ms": 71.209, "p95_ms": 69.532}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.745, "count": 145, "max_ms": 92.404, "p95_ms": 88.403}}, "truncated_count": 0}, "processing_total": {"avg_ms": 196.819, "count": 146, "max_ms": 274.794, "p95_ms": 228.396}, "read_interval": {"avg_ms": 206.347, "count": 145, "max_ms": 286.265, "p95_ms": 238.638}, "triton_wait": {"avg_ms": 67.283, "count": 145, "max_ms": 94.183, "p95_ms": 89.129}, "visualize": {"avg_ms": 8.998, "count": 145, "max_ms": 12.097, "p95_ms": 10.279}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3912.02, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:36:19+09:00", "last_error_summary": "[h264 @ 0x5623024f5900] error while decoding MB 46 19, bytestream -9", "last_exit_at": "2026-07-01T15:36:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.912, "last_started_at": "2026-07-01T15:34:14+09:00", "pid": 5316, "process": {"exists": false, "pid": 5316}, "restart_count": 23, "restart_count_per_hour": 23, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.389, "sample_count": 1398, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:36:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:36:20+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T15:36:19+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T15:36:19.863505+09:00", "mtime_epoch": 1782887779.8635051, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199834}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:36:20+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 866.574, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.872, "pushed": 4.872, "read": 4.839}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.392, "count": 145, "max_ms": 10.925, "p95_ms": 7.87}, "gstreamer_push": {"avg_ms": 0.911, "count": 146, "max_ms": 3.055, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.936, "count": 145, "max_ms": 69.864, "p95_ms": 62.664}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.264, "count": 145, "max_ms": 94.171, "p95_ms": 86.148}, "yolo11_640_fire_11class": {"avg_ms": 53.448, "count": 145, "max_ms": 71.209, "p95_ms": 69.532}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.745, "count": 145, "max_ms": 92.404, "p95_ms": 88.403}}, "truncated_count": 0}, "processing_total": {"avg_ms": 196.819, "count": 146, "max_ms": 274.794, "p95_ms": 228.396}, "read_interval": {"avg_ms": 206.347, "count": 145, "max_ms": 286.265, "p95_ms": 238.638}, "triton_wait": {"avg_ms": 67.283, "count": 145, "max_ms": 94.183, "p95_ms": 89.129}, "visualize": {"avg_ms": 8.998, "count": 145, "max_ms": 12.097, "p95_ms": 10.279}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 102}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4019.441, "limit_mb": 8796093022207.996}, "detail": {"pid": 5423, "restart_count": 24, "start_count": 25}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:36:19+09:00", "last_error_summary": "[h264 @ 0x5623024f5900] error while decoding MB 46 19, bytestream -9", "last_exit_at": "2026-07-01T15:36:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:36:22+09:00", "pid": 5423, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.258, "name": "ffmpeg", "pid": 5423, "rss_mb": 139.258, "state": "S (sleeping)", "threads": 106}, "restart_count": 24, "restart_count_per_hour": 24, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.401, "sample_count": 1374, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:36:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.809, "last_frame_pushed_at": "2026-07-01T15:36:20+09:00", "last_frame_read_age_sec": 5.037, "last_frame_read_at": "2026-07-01T15:36:19+09:00", "mjpeg": {"age_sec": 4.79, "exists": true, "mtime": "2026-07-01T15:36:20.099510+09:00", "mtime_epoch": 1782887780.0995102, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200487}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:36:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 866.574, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.076, "pushed": 4.076, "read": 4.043}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.449, "count": 121, "max_ms": 10.925, "p95_ms": 7.879}, "gstreamer_push": {"avg_ms": 0.912, "count": 122, "max_ms": 3.055, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.22, "count": 121, "max_ms": 69.864, "p95_ms": 61.513}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.484, "count": 121, "max_ms": 94.171, "p95_ms": 85.898}, "yolo11_640_fire_11class": {"avg_ms": 53.64, "count": 121, "max_ms": 71.209, "p95_ms": 68.629}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.077, "count": 121, "max_ms": 92.404, "p95_ms": 88.403}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.701, "count": 122, "max_ms": 274.794, "p95_ms": 229.148}, "read_interval": {"avg_ms": 207.211, "count": 121, "max_ms": 286.265, "p95_ms": 238.638}, "triton_wait": {"avg_ms": 67.58, "count": 121, "max_ms": 94.183, "p95_ms": 89.129}, "visualize": {"avg_ms": 9.046, "count": 121, "max_ms": 12.097, "p95_ms": 10.441}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 107}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3913.84, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556a4cbfbac0] error while decoding MB 40 35, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:38:29+09:00", "last_error_summary": "[h264 @ 0x556a4cbfbac0] error while decoding MB 40 35, bytestream -14", "last_exit_at": "2026-07-01T15:36:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.61, "last_started_at": "2026-07-01T15:36:22+09:00", "pid": 5423, "process": {"exists": false, "pid": 5423}, "restart_count": 24, "restart_count_per_hour": 24, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.389, "sample_count": 1397, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12895, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:38:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T15:38:29+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T15:38:29+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T15:38:29.454294+09:00", "mtime_epoch": 1782887909.4542944, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196887}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:38:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 863.91, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 150, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.015, "pushed": 5.015, "read": 5.015}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.121, "count": 149, "max_ms": 10.804, "p95_ms": 8.233}, "gstreamer_push": {"avg_ms": 0.935, "count": 150, "max_ms": 2.799, "p95_ms": 1.799}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.226, "count": 149, "max_ms": 70.14, "p95_ms": 61.49}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.914, "count": 149, "max_ms": 91.744, "p95_ms": 86.862}, "yolo11_640_fire_11class": {"avg_ms": 53.14, "count": 149, "max_ms": 72.628, "p95_ms": 69.819}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.162, "count": 149, "max_ms": 92.126, "p95_ms": 88.94}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.039, "count": 150, "max_ms": 259.157, "p95_ms": 219.402}, "read_interval": {"avg_ms": 200.465, "count": 150, "max_ms": 269.376, "p95_ms": 228.662}, "triton_wait": {"avg_ms": 66.683, "count": 149, "max_ms": 92.14, "p95_ms": 89.495}, "visualize": {"avg_ms": 8.819, "count": 149, "max_ms": 10.704, "p95_ms": 9.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 107}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3913.195, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:38:29+09:00", "last_error_summary": "[h264 @ 0x556a4cbfbac0] error while decoding MB 40 35, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:38:29+09:00", "last_error_summary": "[h264 @ 0x556a4cbfbac0] error while decoding MB 40 35, bytestream -14", "last_exit_at": "2026-07-01T15:38:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.775, "last_started_at": "2026-07-01T15:36:22+09:00", "pid": 5423, "process": {"exists": false, "pid": 5423}, "restart_count": 24, "restart_count_per_hour": 24, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.389, "sample_count": 1397, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12896, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:38:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:38:29+09:00", "last_frame_read_age_sec": 0.216, "last_frame_read_at": "2026-07-01T15:38:29+09:00", "mjpeg": {"age_sec": 0.208, "exists": true, "mtime": "2026-07-01T15:38:29.454294+09:00", "mtime_epoch": 1782887909.4542944, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196887}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:38:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 863.91, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.021, "pushed": 5.021, "read": 4.988}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.119, "count": 150, "max_ms": 10.804, "p95_ms": 8.233}, "gstreamer_push": {"avg_ms": 0.929, "count": 150, "max_ms": 2.799, "p95_ms": 1.784}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.377, "count": 150, "max_ms": 70.14, "p95_ms": 61.975}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.04, "count": 150, "max_ms": 91.744, "p95_ms": 86.862}, "yolo11_640_fire_11class": {"avg_ms": 53.248, "count": 150, "max_ms": 72.628, "p95_ms": 69.819}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.305, "count": 150, "max_ms": 92.126, "p95_ms": 88.94}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.034, "count": 150, "max_ms": 259.157, "p95_ms": 219.402}, "read_interval": {"avg_ms": 200.315, "count": 149, "max_ms": 269.376, "p95_ms": 228.662}, "triton_wait": {"avg_ms": 66.823, "count": 150, "max_ms": 92.14, "p95_ms": 89.495}, "visualize": {"avg_ms": 8.819, "count": 150, "max_ms": 10.704, "p95_ms": 9.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.875, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 107}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3913.188, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:38:29+09:00", "last_error_summary": "[h264 @ 0x556a4cbfbac0] error while decoding MB 40 35, bytestream -14", "last_exit_at": "2026-07-01T15:38:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.778, "last_started_at": "2026-07-01T15:36:22+09:00", "pid": 5423, "process": {"exists": false, "pid": 5423}, "restart_count": 24, "restart_count_per_hour": 24, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.389, "sample_count": 1397, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12896, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:38:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T15:38:29+09:00", "last_frame_read_age_sec": 0.22, "last_frame_read_at": "2026-07-01T15:38:29+09:00", "mjpeg": {"age_sec": 0.212, "exists": true, "mtime": "2026-07-01T15:38:29.454294+09:00", "mtime_epoch": 1782887909.4542944, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196887}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:38:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 863.91, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.02, "pushed": 5.02, "read": 4.987}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.119, "count": 150, "max_ms": 10.804, "p95_ms": 8.233}, "gstreamer_push": {"avg_ms": 0.929, "count": 150, "max_ms": 2.799, "p95_ms": 1.784}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.377, "count": 150, "max_ms": 70.14, "p95_ms": 61.975}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.04, "count": 150, "max_ms": 91.744, "p95_ms": 86.862}, "yolo11_640_fire_11class": {"avg_ms": 53.248, "count": 150, "max_ms": 72.628, "p95_ms": 69.819}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.305, "count": 150, "max_ms": 92.126, "p95_ms": 88.94}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.034, "count": 150, "max_ms": 259.157, "p95_ms": 219.402}, "read_interval": {"avg_ms": 200.315, "count": 149, "max_ms": 269.376, "p95_ms": 228.662}, "triton_wait": {"avg_ms": 66.823, "count": 150, "max_ms": 92.14, "p95_ms": 89.495}, "visualize": {"avg_ms": 8.819, "count": 150, "max_ms": 10.704, "p95_ms": 9.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 107}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4017.625, "limit_mb": 8796093022207.996}, "detail": {"pid": 5530, "restart_count": 25, "start_count": 26}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:38:29+09:00", "last_error_summary": "[h264 @ 0x556a4cbfbac0] error while decoding MB 40 35, bytestream -14", "last_exit_at": "2026-07-01T15:38:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:38:32+09:00", "pid": 5530, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.867, "name": "ffmpeg", "pid": 5530, "rss_mb": 138.867, "state": "S (sleeping)", "threads": 106}, "restart_count": 25, "restart_count_per_hour": 25, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.389, "sample_count": 1375, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12896, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:38:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.679, "last_frame_pushed_at": "2026-07-01T15:38:29+09:00", "last_frame_read_age_sec": 4.892, "last_frame_read_at": "2026-07-01T15:38:29+09:00", "mjpeg": {"age_sec": 4.664, "exists": true, "mtime": "2026-07-01T15:38:29.674299+09:00", "mtime_epoch": 1782887909.674299, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196548}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:38:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 863.91, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.25, "pushed": 4.25, "read": 4.217}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.161, "count": 127, "max_ms": 10.804, "p95_ms": 8.305}, "gstreamer_push": {"avg_ms": 0.925, "count": 127, "max_ms": 2.799, "p95_ms": 1.784}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.175, "count": 127, "max_ms": 67.852, "p95_ms": 61.975}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.096, "count": 127, "max_ms": 91.744, "p95_ms": 86.862}, "yolo11_640_fire_11class": {"avg_ms": 53.088, "count": 127, "max_ms": 72.628, "p95_ms": 69.68}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.122, "count": 127, "max_ms": 91.564, "p95_ms": 87.858}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.618, "count": 127, "max_ms": 237.036, "p95_ms": 215.547}, "read_interval": {"avg_ms": 199.814, "count": 126, "max_ms": 248.317, "p95_ms": 224.949}, "triton_wait": {"avg_ms": 66.711, "count": 127, "max_ms": 91.757, "p95_ms": 89.122}, "visualize": {"avg_ms": 8.836, "count": 127, "max_ms": 10.704, "p95_ms": 9.782}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 112}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3914.484, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c741224340] error while decoding MB 34 14, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:40:37+09:00", "last_error_summary": "[h264 @ 0x55c741224340] error while decoding MB 34 14, bytestream -5", "last_exit_at": "2026-07-01T15:38:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.028, "last_started_at": "2026-07-01T15:38:32+09:00", "pid": 5530, "process": {"exists": false, "pid": 5530}, "restart_count": 25, "restart_count_per_hour": 25, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.958, "sample_count": 1409, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13493, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:40:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-01T15:40:37+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-01T15:40:37+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-01T15:40:37.312924+09:00", "mtime_epoch": 1782888037.312924, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196173}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:40:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 866.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 144, "pushed_frames": 144, "read_frames": 144, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.816, "pushed": 4.816, "read": 4.816}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.446, "count": 143, "max_ms": 11.26, "p95_ms": 8.168}, "gstreamer_push": {"avg_ms": 0.943, "count": 144, "max_ms": 2.906, "p95_ms": 1.786}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.336, "count": 143, "max_ms": 71.926, "p95_ms": 65.279}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.982, "count": 143, "max_ms": 95.607, "p95_ms": 87.247}, "yolo11_640_fire_11class": {"avg_ms": 55.357, "count": 143, "max_ms": 79.013, "p95_ms": 71.119}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.853, "count": 143, "max_ms": 92.832, "p95_ms": 88.864}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.861, "count": 144, "max_ms": 257.37, "p95_ms": 231.967}, "read_interval": {"avg_ms": 208.674, "count": 144, "max_ms": 268.886, "p95_ms": 241.269}, "triton_wait": {"avg_ms": 69.349, "count": 143, "max_ms": 95.619, "p95_ms": 89.498}, "visualize": {"avg_ms": 9.11, "count": 143, "max_ms": 13.605, "p95_ms": 10.183}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 112}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3904.883, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:40:37+09:00", "last_error_summary": "[h264 @ 0x55c741224340] error while decoding MB 34 14, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:40:37+09:00", "last_error_summary": "[h264 @ 0x55c741224340] error while decoding MB 34 14, bytestream -5", "last_exit_at": "2026-07-01T15:40:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.17, "last_started_at": "2026-07-01T15:38:32+09:00", "pid": 5530, "process": {"exists": false, "pid": 5530}, "restart_count": 25, "restart_count_per_hour": 25, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.958, "sample_count": 1409, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:40:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:40:37+09:00", "last_frame_read_age_sec": 0.201, "last_frame_read_at": "2026-07-01T15:40:37+09:00", "mjpeg": {"age_sec": 0.194, "exists": true, "mtime": "2026-07-01T15:40:37.312924+09:00", "mtime_epoch": 1782888037.312924, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196173}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:40:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 858.371, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 144, "pushed_frames": 144, "read_frames": 143, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.824, "pushed": 4.824, "read": 4.791}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.468, "count": 144, "max_ms": 11.26, "p95_ms": 8.568}, "gstreamer_push": {"avg_ms": 0.942, "count": 144, "max_ms": 2.906, "p95_ms": 1.786}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.339, "count": 144, "max_ms": 71.926, "p95_ms": 65.279}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.985, "count": 144, "max_ms": 95.607, "p95_ms": 87.247}, "yolo11_640_fire_11class": {"avg_ms": 55.363, "count": 144, "max_ms": 79.013, "p95_ms": 71.119}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.856, "count": 144, "max_ms": 92.832, "p95_ms": 88.864}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.798, "count": 144, "max_ms": 257.37, "p95_ms": 231.967}, "read_interval": {"avg_ms": 208.622, "count": 143, "max_ms": 268.886, "p95_ms": 241.269}, "triton_wait": {"avg_ms": 69.348, "count": 144, "max_ms": 95.619, "p95_ms": 89.498}, "visualize": {"avg_ms": 9.11, "count": 144, "max_ms": 13.605, "p95_ms": 10.183}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.85, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 112}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3904.875, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:40:37+09:00", "last_error_summary": "[h264 @ 0x55c741224340] error while decoding MB 34 14, bytestream -5", "last_exit_at": "2026-07-01T15:40:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.173, "last_started_at": "2026-07-01T15:38:32+09:00", "pid": 5530, "process": {"exists": false, "pid": 5530}, "restart_count": 25, "restart_count_per_hour": 25, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.958, "sample_count": 1409, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:40:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:40:37+09:00", "last_frame_read_age_sec": 0.204, "last_frame_read_at": "2026-07-01T15:40:37+09:00", "mjpeg": {"age_sec": 0.197, "exists": true, "mtime": "2026-07-01T15:40:37.312924+09:00", "mtime_epoch": 1782888037.312924, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196173}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:40:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 858.371, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 144, "pushed_frames": 144, "read_frames": 143, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.824, "pushed": 4.824, "read": 4.79}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.468, "count": 144, "max_ms": 11.26, "p95_ms": 8.568}, "gstreamer_push": {"avg_ms": 0.942, "count": 144, "max_ms": 2.906, "p95_ms": 1.786}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.339, "count": 144, "max_ms": 71.926, "p95_ms": 65.279}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.985, "count": 144, "max_ms": 95.607, "p95_ms": 87.247}, "yolo11_640_fire_11class": {"avg_ms": 55.363, "count": 144, "max_ms": 79.013, "p95_ms": 71.119}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.856, "count": 144, "max_ms": 92.832, "p95_ms": 88.864}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.798, "count": 144, "max_ms": 257.37, "p95_ms": 231.967}, "read_interval": {"avg_ms": 208.622, "count": 143, "max_ms": 268.886, "p95_ms": 241.269}, "triton_wait": {"avg_ms": 69.348, "count": 144, "max_ms": 95.619, "p95_ms": 89.498}, "visualize": {"avg_ms": 9.11, "count": 144, "max_ms": 13.605, "p95_ms": 10.183}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.853, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 112}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4011.156, "limit_mb": 8796093022207.996}, "detail": {"pid": 5637, "restart_count": 26, "start_count": 27}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:40:37+09:00", "last_error_summary": "[h264 @ 0x55c741224340] error while decoding MB 34 14, bytestream -5", "last_exit_at": "2026-07-01T15:40:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:40:39+09:00", "pid": 5637, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.441, "name": "ffmpeg", "pid": 5637, "rss_mb": 139.441, "state": "S (sleeping)", "threads": 106}, "restart_count": 26, "restart_count_per_hour": 26, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 227.958, "sample_count": 1388, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:40:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.429, "last_frame_pushed_at": "2026-07-01T15:40:37+09:00", "last_frame_read_age_sec": 4.627, "last_frame_read_at": "2026-07-01T15:40:37+09:00", "mjpeg": {"age_sec": 4.412, "exists": true, "mtime": "2026-07-01T15:40:37.520928+09:00", "mtime_epoch": 1782888037.5209281, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190199}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:40:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 858.371, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.094, "pushed": 4.094, "read": 4.061}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.502, "count": 122, "max_ms": 11.26, "p95_ms": 8.568}, "gstreamer_push": {"avg_ms": 0.94, "count": 122, "max_ms": 2.906, "p95_ms": 1.782}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.788, "count": 122, "max_ms": 71.926, "p95_ms": 65.638}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.151, "count": 122, "max_ms": 95.607, "p95_ms": 87.247}, "yolo11_640_fire_11class": {"avg_ms": 55.498, "count": 122, "max_ms": 79.013, "p95_ms": 71.119}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.128, "count": 122, "max_ms": 92.832, "p95_ms": 88.96}}, "truncated_count": 0}, "processing_total": {"avg_ms": 199.971, "count": 122, "max_ms": 257.37, "p95_ms": 231.967}, "read_interval": {"avg_ms": 209.8, "count": 121, "max_ms": 268.886, "p95_ms": 241.269}, "triton_wait": {"avg_ms": 69.621, "count": 122, "max_ms": 95.619, "p95_ms": 89.498}, "visualize": {"avg_ms": 9.125, "count": 122, "max_ms": 13.605, "p95_ms": 10.095}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.798, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3913.285, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555f2c4342c0] error while decoding MB 66 11, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:42:47+09:00", "last_error_summary": "[h264 @ 0x555f2c4342c0] error while decoding MB 66 11, bytestream -7", "last_exit_at": "2026-07-01T15:40:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.959, "last_started_at": "2026-07-01T15:40:39+09:00", "pid": 5637, "process": {"exists": false, "pid": 5637}, "restart_count": 26, "restart_count_per_hour": 26, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.249, "sample_count": 1423, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14114, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:42:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T15:42:47+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T15:42:47+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T15:42:47.847511+09:00", "mtime_epoch": 1782888167.8475108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199927}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:42:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.727, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.916, "pushed": 4.916, "read": 4.916}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.146, "count": 146, "max_ms": 9.44, "p95_ms": 7.973}, "gstreamer_push": {"avg_ms": 0.925, "count": 147, "max_ms": 1.892, "p95_ms": 1.762}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.355, "count": 146, "max_ms": 67.467, "p95_ms": 64.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.066, "count": 146, "max_ms": 95.993, "p95_ms": 87.598}, "yolo11_640_fire_11class": {"avg_ms": 54.899, "count": 146, "max_ms": 84.803, "p95_ms": 70.392}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.778, "count": 146, "max_ms": 104.369, "p95_ms": 88.149}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.54, "count": 147, "max_ms": 264.198, "p95_ms": 225.816}, "read_interval": {"avg_ms": 204.285, "count": 147, "max_ms": 275.824, "p95_ms": 235.993}, "triton_wait": {"avg_ms": 68.583, "count": 146, "max_ms": 104.382, "p95_ms": 89.567}, "visualize": {"avg_ms": 8.976, "count": 146, "max_ms": 13.412, "p95_ms": 10.053}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3915.016, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:42:47+09:00", "last_error_summary": "[h264 @ 0x555f2c4342c0] error while decoding MB 66 11, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:42:47+09:00", "last_error_summary": "[h264 @ 0x555f2c4342c0] error while decoding MB 66 11, bytestream -7", "last_exit_at": "2026-07-01T15:42:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.126, "last_started_at": "2026-07-01T15:40:39+09:00", "pid": 5637, "process": {"exists": false, "pid": 5637}, "restart_count": 26, "restart_count_per_hour": 26, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.249, "sample_count": 1424, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14115, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:42:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:42:48+09:00", "last_frame_read_age_sec": 0.218, "last_frame_read_at": "2026-07-01T15:42:47+09:00", "mjpeg": {"age_sec": 0.21, "exists": true, "mtime": "2026-07-01T15:42:47.847511+09:00", "mtime_epoch": 1782888167.8475108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199927}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:42:48+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.727, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.919, "pushed": 4.919, "read": 4.886}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.151, "count": 147, "max_ms": 9.44, "p95_ms": 7.973}, "gstreamer_push": {"avg_ms": 0.926, "count": 147, "max_ms": 1.892, "p95_ms": 1.762}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.297, "count": 147, "max_ms": 67.467, "p95_ms": 64.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.023, "count": 147, "max_ms": 95.993, "p95_ms": 87.598}, "yolo11_640_fire_11class": {"avg_ms": 54.892, "count": 147, "max_ms": 84.803, "p95_ms": 70.392}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.738, "count": 147, "max_ms": 104.369, "p95_ms": 88.149}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.767, "count": 147, "max_ms": 264.198, "p95_ms": 225.816}, "read_interval": {"avg_ms": 204.385, "count": 146, "max_ms": 275.824, "p95_ms": 235.993}, "triton_wait": {"avg_ms": 68.537, "count": 147, "max_ms": 104.382, "p95_ms": 89.567}, "visualize": {"avg_ms": 8.98, "count": 147, "max_ms": 13.412, "p95_ms": 10.053}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3915.008, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:42:47+09:00", "last_error_summary": "[h264 @ 0x555f2c4342c0] error while decoding MB 66 11, bytestream -7", "last_exit_at": "2026-07-01T15:42:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.129, "last_started_at": "2026-07-01T15:40:39+09:00", "pid": 5637, "process": {"exists": false, "pid": 5637}, "restart_count": 26, "restart_count_per_hour": 26, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.249, "sample_count": 1424, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14115, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:42:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T15:42:48+09:00", "last_frame_read_age_sec": 0.221, "last_frame_read_at": "2026-07-01T15:42:47+09:00", "mjpeg": {"age_sec": 0.214, "exists": true, "mtime": "2026-07-01T15:42:47.847511+09:00", "mtime_epoch": 1782888167.8475108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199927}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:42:48+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.727, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.919, "pushed": 4.919, "read": 4.885}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.151, "count": 147, "max_ms": 9.44, "p95_ms": 7.973}, "gstreamer_push": {"avg_ms": 0.926, "count": 147, "max_ms": 1.892, "p95_ms": 1.762}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.297, "count": 147, "max_ms": 67.467, "p95_ms": 64.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.023, "count": 147, "max_ms": 95.993, "p95_ms": 87.598}, "yolo11_640_fire_11class": {"avg_ms": 54.892, "count": 147, "max_ms": 84.803, "p95_ms": 70.392}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.738, "count": 147, "max_ms": 104.369, "p95_ms": 88.149}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.767, "count": 147, "max_ms": 264.198, "p95_ms": 225.816}, "read_interval": {"avg_ms": 204.385, "count": 146, "max_ms": 275.824, "p95_ms": 235.993}, "triton_wait": {"avg_ms": 68.537, "count": 147, "max_ms": 104.382, "p95_ms": 89.567}, "visualize": {"avg_ms": 8.98, "count": 147, "max_ms": 13.412, "p95_ms": 10.053}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 117}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4023.449, "limit_mb": 8796093022207.996}, "detail": {"pid": 5744, "restart_count": 27, "start_count": 28}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:42:47+09:00", "last_error_summary": "[h264 @ 0x555f2c4342c0] error while decoding MB 66 11, bytestream -7", "last_exit_at": "2026-07-01T15:42:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:42:50+09:00", "pid": 5744, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.574, "name": "ffmpeg", "pid": 5744, "rss_mb": 136.574, "state": "S (sleeping)", "threads": 106}, "restart_count": 27, "restart_count_per_hour": 27, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.967, "sample_count": 1404, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14115, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:42:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.255, "last_frame_pushed_at": "2026-07-01T15:42:48+09:00", "last_frame_read_age_sec": 4.47, "last_frame_read_at": "2026-07-01T15:42:47+09:00", "mjpeg": {"age_sec": 4.238, "exists": true, "mtime": "2026-07-01T15:42:48.071515+09:00", "mtime_epoch": 1782888168.0715153, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200496}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:42:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.727, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.215, "pushed": 4.215, "read": 4.182}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.176, "count": 126, "max_ms": 9.44, "p95_ms": 7.991}, "gstreamer_push": {"avg_ms": 0.929, "count": 126, "max_ms": 1.892, "p95_ms": 1.801}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.192, "count": 126, "max_ms": 67.467, "p95_ms": 64.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.664, "count": 126, "max_ms": 95.993, "p95_ms": 87.223}, "yolo11_640_fire_11class": {"avg_ms": 54.858, "count": 126, "max_ms": 84.803, "p95_ms": 69.709}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.699, "count": 126, "max_ms": 104.369, "p95_ms": 88.149}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.425, "count": 126, "max_ms": 264.198, "p95_ms": 225.816}, "read_interval": {"avg_ms": 205.036, "count": 125, "max_ms": 275.824, "p95_ms": 235.993}, "triton_wait": {"avg_ms": 68.462, "count": 126, "max_ms": 104.382, "p95_ms": 89.567}, "visualize": {"avg_ms": 9.041, "count": 126, "max_ms": 13.412, "p95_ms": 10.149}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3916.871, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55bdd120ae40] error while decoding MB 40 40, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:44:59+09:00", "last_error_summary": "[h264 @ 0x55bdd120ae40] error while decoding MB 40 40, bytestream -5", "last_exit_at": "2026-07-01T15:42:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.98, "last_started_at": "2026-07-01T15:42:50+09:00", "pid": 5744, "process": {"exists": false, "pid": 5744}, "restart_count": 27, "restart_count_per_hour": 27, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.149, "sample_count": 1432, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14741, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:44:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-01T15:44:59+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T15:44:59+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T15:44:59.250038+09:00", "mtime_epoch": 1782888299.250038, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189459}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:44:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 867.398, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.901, "pushed": 4.901, "read": 4.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.051, "count": 147, "max_ms": 8.366, "p95_ms": 7.917}, "gstreamer_push": {"avg_ms": 0.965, "count": 147, "max_ms": 3.04, "p95_ms": 1.799}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.261, "count": 147, "max_ms": 71.056, "p95_ms": 66.371}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.283, "count": 147, "max_ms": 92.505, "p95_ms": 88.919}, "yolo11_640_fire_11class": {"avg_ms": 55.636, "count": 147, "max_ms": 76.281, "p95_ms": 71.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.594, "count": 147, "max_ms": 95.522, "p95_ms": 90.117}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.129, "count": 147, "max_ms": 251.772, "p95_ms": 216.445}, "read_interval": {"avg_ms": 203.716, "count": 148, "max_ms": 261.978, "p95_ms": 227.225}, "triton_wait": {"avg_ms": 70.137, "count": 147, "max_ms": 95.534, "p95_ms": 91.058}, "visualize": {"avg_ms": 8.775, "count": 147, "max_ms": 11.785, "p95_ms": 9.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3908.105, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:44:59+09:00", "last_error_summary": "[h264 @ 0x55bdd120ae40] error while decoding MB 40 40, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:44:59+09:00", "last_error_summary": "[h264 @ 0x55bdd120ae40] error while decoding MB 40 40, bytestream -5", "last_exit_at": "2026-07-01T15:44:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.123, "last_started_at": "2026-07-01T15:42:50+09:00", "pid": 5744, "process": {"exists": false, "pid": 5744}, "restart_count": 27, "restart_count_per_hour": 27, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.149, "sample_count": 1433, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14742, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:44:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:44:59+09:00", "last_frame_read_age_sec": 0.193, "last_frame_read_at": "2026-07-01T15:44:59+09:00", "mjpeg": {"age_sec": 0.182, "exists": true, "mtime": "2026-07-01T15:44:59.250038+09:00", "mtime_epoch": 1782888299.250038, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189459}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:44:59+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 867.398, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.942, "pushed": 4.942, "read": 4.908}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.053, "count": 147, "max_ms": 8.366, "p95_ms": 7.917}, "gstreamer_push": {"avg_ms": 0.965, "count": 148, "max_ms": 3.04, "p95_ms": 1.799}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.309, "count": 147, "max_ms": 71.056, "p95_ms": 66.371}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.299, "count": 147, "max_ms": 92.505, "p95_ms": 88.919}, "yolo11_640_fire_11class": {"avg_ms": 55.675, "count": 147, "max_ms": 76.281, "p95_ms": 71.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.614, "count": 147, "max_ms": 95.522, "p95_ms": 90.117}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.089, "count": 148, "max_ms": 251.772, "p95_ms": 216.445}, "read_interval": {"avg_ms": 203.712, "count": 147, "max_ms": 261.978, "p95_ms": 227.225}, "triton_wait": {"avg_ms": 70.156, "count": 147, "max_ms": 95.534, "p95_ms": 91.058}, "visualize": {"avg_ms": 8.772, "count": 147, "max_ms": 11.785, "p95_ms": 9.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3908.098, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:44:59+09:00", "last_error_summary": "[h264 @ 0x55bdd120ae40] error while decoding MB 40 40, bytestream -5", "last_exit_at": "2026-07-01T15:44:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.126, "last_started_at": "2026-07-01T15:42:50+09:00", "pid": 5744, "process": {"exists": false, "pid": 5744}, "restart_count": 27, "restart_count_per_hour": 27, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.149, "sample_count": 1433, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14742, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:44:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T15:44:59+09:00", "last_frame_read_age_sec": 0.196, "last_frame_read_at": "2026-07-01T15:44:59+09:00", "mjpeg": {"age_sec": 0.185, "exists": true, "mtime": "2026-07-01T15:44:59.250038+09:00", "mtime_epoch": 1782888299.250038, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189459}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:44:59+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 867.398, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.941, "pushed": 4.941, "read": 4.908}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.053, "count": 147, "max_ms": 8.366, "p95_ms": 7.917}, "gstreamer_push": {"avg_ms": 0.965, "count": 148, "max_ms": 3.04, "p95_ms": 1.799}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.309, "count": 147, "max_ms": 71.056, "p95_ms": 66.371}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.299, "count": 147, "max_ms": 92.505, "p95_ms": 88.919}, "yolo11_640_fire_11class": {"avg_ms": 55.675, "count": 147, "max_ms": 76.281, "p95_ms": 71.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.614, "count": 147, "max_ms": 95.522, "p95_ms": 90.117}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.089, "count": 148, "max_ms": 251.772, "p95_ms": 216.445}, "read_interval": {"avg_ms": 203.712, "count": 147, "max_ms": 261.978, "p95_ms": 227.225}, "triton_wait": {"avg_ms": 70.156, "count": 147, "max_ms": 95.534, "p95_ms": 91.058}, "visualize": {"avg_ms": 8.772, "count": 147, "max_ms": 11.785, "p95_ms": 9.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 122}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4027.281, "limit_mb": 8796093022207.996}, "detail": {"pid": 5851, "restart_count": 28, "start_count": 29}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:44:59+09:00", "last_error_summary": "[h264 @ 0x55bdd120ae40] error while decoding MB 40 40, bytestream -5", "last_exit_at": "2026-07-01T15:45:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:45:01+09:00", "pid": 5851, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.844, "name": "ffmpeg", "pid": 5851, "rss_mb": 141.844, "state": "S (sleeping)", "threads": 106}, "restart_count": 28, "restart_count_per_hour": 28, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.272, "sample_count": 1412, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 14742, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:44:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.051, "last_frame_pushed_at": "2026-07-01T15:44:59+09:00", "last_frame_read_age_sec": 4.241, "last_frame_read_at": "2026-07-01T15:44:59+09:00", "mjpeg": {"age_sec": 4.034, "exists": true, "mtime": "2026-07-01T15:44:59.446042+09:00", "mtime_epoch": 1782888299.4460416, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:45:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 867.398, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.28, "pushed": 4.28, "read": 4.247}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.067, "count": 127, "max_ms": 8.366, "p95_ms": 7.921}, "gstreamer_push": {"avg_ms": 0.972, "count": 128, "max_ms": 3.04, "p95_ms": 1.812}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.057, "count": 127, "max_ms": 71.056, "p95_ms": 66.371}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.524, "count": 127, "max_ms": 91.874, "p95_ms": 88.796}, "yolo11_640_fire_11class": {"avg_ms": 55.477, "count": 127, "max_ms": 76.281, "p95_ms": 71.024}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.393, "count": 127, "max_ms": 95.522, "p95_ms": 90.117}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.027, "count": 128, "max_ms": 251.772, "p95_ms": 216.445}, "read_interval": {"avg_ms": 203.612, "count": 127, "max_ms": 261.978, "p95_ms": 227.225}, "triton_wait": {"avg_ms": 69.639, "count": 127, "max_ms": 95.534, "p95_ms": 91.058}, "visualize": {"avg_ms": 8.787, "count": 127, "max_ms": 11.785, "p95_ms": 10.011}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 128}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4171.242, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55be3df9d880] error while decoding MB 28 6, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:47:05+09:00", "last_error_summary": "[h264 @ 0x55be3df9d880] error while decoding MB 28 6, bytestream -5", "last_exit_at": "2026-07-01T15:45:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.716, "last_started_at": "2026-07-01T15:45:01+09:00", "pid": 5851, "process": {"exists": false, "pid": 5851}, "restart_count": 28, "restart_count_per_hour": 28, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.403, "sample_count": 1461, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15364, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:47:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T15:47:05+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T15:47:05+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T15:47:05.152403+09:00", "mtime_epoch": 1782888425.1524029, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201578}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:47:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.531, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 151, "pushed_frames": 151, "read_frames": 151, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.065, "pushed": 5.065, "read": 5.065}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.261, "count": 151, "max_ms": 9.108, "p95_ms": 8.579}, "gstreamer_push": {"avg_ms": 0.95, "count": 151, "max_ms": 2.912, "p95_ms": 1.75}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.254, "count": 151, "max_ms": 78.037, "p95_ms": 71.823}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.739, "count": 151, "max_ms": 97.438, "p95_ms": 93.242}, "yolo11_640_fire_11class": {"avg_ms": 51.894, "count": 151, "max_ms": 76.904, "p95_ms": 73.11}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.206, "count": 151, "max_ms": 98.771, "p95_ms": 95.816}}, "truncated_count": 0}, "processing_total": {"avg_ms": 188.831, "count": 151, "max_ms": 236.456, "p95_ms": 223.577}, "read_interval": {"avg_ms": 198.377, "count": 151, "max_ms": 246.168, "p95_ms": 232.808}, "triton_wait": {"avg_ms": 65.402, "count": 151, "max_ms": 98.785, "p95_ms": 95.828}, "visualize": {"avg_ms": 8.824, "count": 151, "max_ms": 11.503, "p95_ms": 10.362}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.81, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 128}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4170.391, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:47:05+09:00", "last_error_summary": "[h264 @ 0x55be3df9d880] error while decoding MB 28 6, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:47:05+09:00", "last_error_summary": "[h264 @ 0x55be3df9d880] error while decoding MB 28 6, bytestream -5", "last_exit_at": "2026-07-01T15:47:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.857, "last_started_at": "2026-07-01T15:45:01+09:00", "pid": 5851, "process": {"exists": false, "pid": 5851}, "restart_count": 28, "restart_count_per_hour": 28, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.403, "sample_count": 1461, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15365, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:47:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:47:05+09:00", "last_frame_read_age_sec": 0.194, "last_frame_read_at": "2026-07-01T15:47:05+09:00", "mjpeg": {"age_sec": 0.184, "exists": true, "mtime": "2026-07-01T15:47:05.152403+09:00", "mtime_epoch": 1782888425.1524029, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201578}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:47:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.531, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 151, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.075, "pushed": 5.075, "read": 5.041}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.258, "count": 151, "max_ms": 9.108, "p95_ms": 8.579}, "gstreamer_push": {"avg_ms": 0.95, "count": 152, "max_ms": 2.912, "p95_ms": 1.75}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.276, "count": 151, "max_ms": 78.037, "p95_ms": 71.823}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.677, "count": 151, "max_ms": 97.438, "p95_ms": 93.242}, "yolo11_640_fire_11class": {"avg_ms": 51.793, "count": 151, "max_ms": 75.267, "p95_ms": 72.073}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.11, "count": 151, "max_ms": 98.771, "p95_ms": 95.803}}, "truncated_count": 0}, "processing_total": {"avg_ms": 188.835, "count": 152, "max_ms": 236.456, "p95_ms": 223.577}, "read_interval": {"avg_ms": 198.377, "count": 151, "max_ms": 246.168, "p95_ms": 232.808}, "triton_wait": {"avg_ms": 65.306, "count": 151, "max_ms": 98.785, "p95_ms": 95.815}, "visualize": {"avg_ms": 8.823, "count": 151, "max_ms": 11.503, "p95_ms": 10.362}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 128}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4170.383, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:47:05+09:00", "last_error_summary": "[h264 @ 0x55be3df9d880] error while decoding MB 28 6, bytestream -5", "last_exit_at": "2026-07-01T15:47:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.861, "last_started_at": "2026-07-01T15:45:01+09:00", "pid": 5851, "process": {"exists": false, "pid": 5851}, "restart_count": 28, "restart_count_per_hour": 28, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.403, "sample_count": 1461, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15365, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:47:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T15:47:05+09:00", "last_frame_read_age_sec": 0.198, "last_frame_read_at": "2026-07-01T15:47:05+09:00", "mjpeg": {"age_sec": 0.188, "exists": true, "mtime": "2026-07-01T15:47:05.152403+09:00", "mtime_epoch": 1782888425.1524029, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201578}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:47:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.531, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 151, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.074, "pushed": 5.074, "read": 5.041}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.258, "count": 151, "max_ms": 9.108, "p95_ms": 8.579}, "gstreamer_push": {"avg_ms": 0.95, "count": 152, "max_ms": 2.912, "p95_ms": 1.75}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.276, "count": 151, "max_ms": 78.037, "p95_ms": 71.823}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.677, "count": 151, "max_ms": 97.438, "p95_ms": 93.242}, "yolo11_640_fire_11class": {"avg_ms": 51.793, "count": 151, "max_ms": 75.267, "p95_ms": 72.073}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.11, "count": 151, "max_ms": 98.771, "p95_ms": 95.803}}, "truncated_count": 0}, "processing_total": {"avg_ms": 188.835, "count": 152, "max_ms": 236.456, "p95_ms": 223.577}, "read_interval": {"avg_ms": 198.377, "count": 151, "max_ms": 246.168, "p95_ms": 232.808}, "triton_wait": {"avg_ms": 65.306, "count": 151, "max_ms": 98.785, "p95_ms": 95.815}, "visualize": {"avg_ms": 8.823, "count": 151, "max_ms": 11.503, "p95_ms": 10.362}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 128}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4263.062, "limit_mb": 8796093022207.996}, "detail": {"pid": 6018, "restart_count": 29, "start_count": 30}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:47:05+09:00", "last_error_summary": "[h264 @ 0x55be3df9d880] error while decoding MB 28 6, bytestream -5", "last_exit_at": "2026-07-01T15:47:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:47:08+09:00", "pid": 6018, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.707, "name": "ffmpeg", "pid": 6018, "rss_mb": 138.707, "state": "S (sleeping)", "threads": 106}, "restart_count": 29, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.577, "sample_count": 1436, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15365, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:47:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.978, "last_frame_pushed_at": "2026-07-01T15:47:05+09:00", "last_frame_read_age_sec": 5.169, "last_frame_read_at": "2026-07-01T15:47:05+09:00", "mjpeg": {"age_sec": 4.963, "exists": true, "mtime": "2026-07-01T15:47:05.348407+09:00", "mtime_epoch": 1782888425.3484066, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192021}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:47:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.531, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.208, "pushed": 4.208, "read": 4.174}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.245, "count": 125, "max_ms": 9.108, "p95_ms": 8.493}, "gstreamer_push": {"avg_ms": 0.932, "count": 126, "max_ms": 1.977, "p95_ms": 1.746}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.584, "count": 125, "max_ms": 78.037, "p95_ms": 68.817}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.324, "count": 125, "max_ms": 97.438, "p95_ms": 93.242}, "yolo11_640_fire_11class": {"avg_ms": 52.025, "count": 125, "max_ms": 75.267, "p95_ms": 72.073}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.735, "count": 125, "max_ms": 98.771, "p95_ms": 95.803}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.267, "count": 126, "max_ms": 236.456, "p95_ms": 224.144}, "read_interval": {"avg_ms": 199.722, "count": 125, "max_ms": 246.168, "p95_ms": 234.807}, "triton_wait": {"avg_ms": 65.949, "count": 125, "max_ms": 98.785, "p95_ms": 95.815}, "visualize": {"avg_ms": 8.825, "count": 125, "max_ms": 11.503, "p95_ms": 10.358}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 286}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4085.02, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56348e8eba80] error while decoding MB 93 29, bytestream -39 [in#0/rtsp @ 0x56348dbd5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56348dbd5e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:48:46+09:00", "last_error_summary": "[h264 @ 0x56348e8eba80] error while decoding MB 93 29, bytestream -39 [in#0/rtsp @ 0x56348dbd5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56348dbd5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:11:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2260.711, "last_started_at": "2026-07-01T15:11:06+09:00", "pid": 3205, "process": {"exists": false, "pid": 3205}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.622, "sample_count": 2564, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34355, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:48:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-01T15:48:46+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-01T15:48:46+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-01T15:48:46.846281+09:00", "mtime_epoch": 1782888526.846281, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257886}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T15:48:46+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1143.77, "name": "celery", "pid": 267, "rss_mb": 1106.203, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 123, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.839, "pushed": 6.839, "read": 6.895}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.054, "count": 122, "max_ms": 2.858, "p95_ms": 2.651}, "gstreamer_push": {"avg_ms": 1.316, "count": 122, "max_ms": 1.562, "p95_ms": 1.433}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.977, "count": 122, "max_ms": 71.372, "p95_ms": 56.228}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.431, "count": 122, "max_ms": 90.298, "p95_ms": 82.985}, "yolo11_640_fire_11class": {"avg_ms": 54.317, "count": 122, "max_ms": 79.079, "p95_ms": 65.188}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.069, "count": 122, "max_ms": 91.317, "p95_ms": 84.439}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.218, "count": 122, "max_ms": 101.857, "p95_ms": 97.343}, "read_interval": {"avg_ms": 300.48, "count": 123, "max_ms": 19180.265, "p95_ms": 113.722}, "triton_wait": {"avg_ms": 59.569, "count": 122, "max_ms": 91.329, "p95_ms": 86.812}, "visualize": {"avg_ms": 3.693, "count": 122, "max_ms": 4.478, "p95_ms": 4.177}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.839, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 286}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4084.211, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:48:46+09:00", "last_error_summary": "[h264 @ 0x56348e8eba80] error while decoding MB 93 29, bytestream -39 [in#0/rtsp @ 0x56348dbd5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56348dbd5e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:48:46+09:00", "last_error_summary": "[h264 @ 0x56348e8eba80] error while decoding MB 93 29, bytestream -39 [in#0/rtsp @ 0x56348dbd5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56348dbd5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:48:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2260.746, "last_started_at": "2026-07-01T15:11:06+09:00", "pid": 3205, "process": {"exists": false, "pid": 3205}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.622, "sample_count": 2565, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34356, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:48:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:48:46+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T15:48:46+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-01T15:48:46.846281+09:00", "mtime_epoch": 1782888526.846281, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257886}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T15:48:46+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1143.77, "name": "celery", "pid": 267, "rss_mb": 1106.219, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.882, "pushed": 6.882, "read": 6.882}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.053, "count": 123, "max_ms": 2.858, "p95_ms": 2.651}, "gstreamer_push": {"avg_ms": 1.315, "count": 123, "max_ms": 1.562, "p95_ms": 1.433}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.075, "count": 123, "max_ms": 71.372, "p95_ms": 56.228}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.569, "count": 123, "max_ms": 90.298, "p95_ms": 82.985}, "yolo11_640_fire_11class": {"avg_ms": 54.351, "count": 123, "max_ms": 79.079, "p95_ms": 65.188}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.23, "count": 123, "max_ms": 91.317, "p95_ms": 84.439}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.349, "count": 123, "max_ms": 101.857, "p95_ms": 97.343}, "read_interval": {"avg_ms": 300.48, "count": 123, "max_ms": 19180.265, "p95_ms": 113.722}, "triton_wait": {"avg_ms": 59.706, "count": 123, "max_ms": 91.329, "p95_ms": 86.812}, "visualize": {"avg_ms": 3.691, "count": 123, "max_ms": 4.478, "p95_ms": 4.177}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.873, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 286}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4084.141, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:48:46+09:00", "last_error_summary": "[h264 @ 0x56348e8eba80] error while decoding MB 93 29, bytestream -39 [in#0/rtsp @ 0x56348dbd5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56348dbd5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:48:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2260.754, "last_started_at": "2026-07-01T15:11:06+09:00", "pid": 3205, "process": {"exists": false, "pid": 3205}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.622, "sample_count": 2565, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34356, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:48:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T15:48:46+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T15:48:46+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-01T15:48:46.846281+09:00", "mtime_epoch": 1782888526.846281, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257886}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T15:48:46+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1143.77, "name": "celery", "pid": 267, "rss_mb": 1106.219, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.879, "pushed": 6.879, "read": 6.879}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.053, "count": 123, "max_ms": 2.858, "p95_ms": 2.651}, "gstreamer_push": {"avg_ms": 1.315, "count": 123, "max_ms": 1.562, "p95_ms": 1.433}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.075, "count": 123, "max_ms": 71.372, "p95_ms": 56.228}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.569, "count": 123, "max_ms": 90.298, "p95_ms": 82.985}, "yolo11_640_fire_11class": {"avg_ms": 54.351, "count": 123, "max_ms": 79.079, "p95_ms": 65.188}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.23, "count": 123, "max_ms": 91.317, "p95_ms": 84.439}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.349, "count": 123, "max_ms": 101.857, "p95_ms": 97.343}, "read_interval": {"avg_ms": 300.48, "count": 123, "max_ms": 19180.265, "p95_ms": 113.722}, "triton_wait": {"avg_ms": 59.706, "count": 123, "max_ms": 91.329, "p95_ms": 86.812}, "visualize": {"avg_ms": 3.691, "count": 123, "max_ms": 4.478, "p95_ms": 4.177}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.881, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 286}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4205.73, "limit_mb": 8796093022207.996}, "detail": {"pid": 6125, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:48:46+09:00", "last_error_summary": "[h264 @ 0x56348e8eba80] error while decoding MB 93 29, bytestream -39 [in#0/rtsp @ 0x56348dbd5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56348dbd5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:48:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:48:49+09:00", "pid": 6125, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.547, "name": "ffmpeg", "pid": 6125, "rss_mb": 156.547, "state": "R (running)", "threads": 49}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.948, "sample_count": 2522, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34356, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:48:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.596, "last_frame_pushed_at": "2026-07-01T15:48:46+09:00", "last_frame_read_age_sec": 4.685, "last_frame_read_at": "2026-07-01T15:48:46+09:00", "mjpeg": {"age_sec": 4.578, "exists": true, "mtime": "2026-07-01T15:48:46.946283+09:00", "mtime_epoch": 1782888526.9462829, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257836}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T15:48:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1143.77, "name": "celery", "pid": 267, "rss_mb": 1106.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.476, "pushed": 5.476, "read": 5.476}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.053, "count": 123, "max_ms": 2.858, "p95_ms": 2.651}, "gstreamer_push": {"avg_ms": 1.315, "count": 123, "max_ms": 1.562, "p95_ms": 1.433}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.075, "count": 123, "max_ms": 71.372, "p95_ms": 56.228}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.569, "count": 123, "max_ms": 90.298, "p95_ms": 82.985}, "yolo11_640_fire_11class": {"avg_ms": 54.351, "count": 123, "max_ms": 79.079, "p95_ms": 65.188}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.23, "count": 123, "max_ms": 91.317, "p95_ms": 84.439}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.349, "count": 123, "max_ms": 101.857, "p95_ms": 97.343}, "read_interval": {"avg_ms": 300.48, "count": 123, "max_ms": 19180.265, "p95_ms": 113.722}, "triton_wait": {"avg_ms": 59.706, "count": 123, "max_ms": 91.329, "p95_ms": 86.812}, "visualize": {"avg_ms": 3.691, "count": 123, "max_ms": 4.478, "p95_ms": 4.177}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.464, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4149.18, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558cd993eac0] error while decoding MB 71 22, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:49:11+09:00", "last_error_summary": "[h264 @ 0x558cd993eac0] error while decoding MB 71 22, bytestream -13", "last_exit_at": "2026-07-01T15:47:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.255, "last_started_at": "2026-07-01T15:47:08+09:00", "pid": 6018, "process": {"exists": false, "pid": 6018}, "restart_count": 29, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.898, "sample_count": 1480, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15985, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:49:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T15:49:11+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T15:49:11+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T15:49:11.522733+09:00", "mtime_epoch": 1782888551.5227332, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196742}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:49:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.871, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.089, "pushed": 5.089, "read": 5.089}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.031, "count": 152, "max_ms": 9.392, "p95_ms": 8.087}, "gstreamer_push": {"avg_ms": 0.941, "count": 152, "max_ms": 2.863, "p95_ms": 1.788}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.528, "count": 152, "max_ms": 69.824, "p95_ms": 63.629}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.351, "count": 152, "max_ms": 95.745, "p95_ms": 88.86}, "yolo11_640_fire_11class": {"avg_ms": 52.761, "count": 152, "max_ms": 77.162, "p95_ms": 72.05}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.868, "count": 152, "max_ms": 95.701, "p95_ms": 91.169}}, "truncated_count": 0}, "processing_total": {"avg_ms": 188.022, "count": 152, "max_ms": 255.042, "p95_ms": 219.397}, "read_interval": {"avg_ms": 197.28, "count": 152, "max_ms": 266.637, "p95_ms": 229.303}, "triton_wait": {"avg_ms": 66.32, "count": 152, "max_ms": 95.756, "p95_ms": 91.708}, "visualize": {"avg_ms": 8.709, "count": 152, "max_ms": 10.76, "p95_ms": 10.019}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4148.285, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:49:11+09:00", "last_error_summary": "[h264 @ 0x558cd993eac0] error while decoding MB 71 22, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:49:11+09:00", "last_error_summary": "[h264 @ 0x558cd993eac0] error while decoding MB 71 22, bytestream -13", "last_exit_at": "2026-07-01T15:49:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.396, "last_started_at": "2026-07-01T15:47:08+09:00", "pid": 6018, "process": {"exists": false, "pid": 6018}, "restart_count": 29, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.898, "sample_count": 1480, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15986, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:49:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:49:11+09:00", "last_frame_read_age_sec": 0.192, "last_frame_read_at": "2026-07-01T15:49:11+09:00", "mjpeg": {"age_sec": 0.183, "exists": true, "mtime": "2026-07-01T15:49:11.522733+09:00", "mtime_epoch": 1782888551.5227332, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196742}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:49:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.871, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 151, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.097, "pushed": 5.097, "read": 5.064}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.024, "count": 152, "max_ms": 9.392, "p95_ms": 8.087}, "gstreamer_push": {"avg_ms": 0.942, "count": 152, "max_ms": 2.863, "p95_ms": 1.788}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.59, "count": 152, "max_ms": 69.824, "p95_ms": 63.629}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.575, "count": 152, "max_ms": 95.745, "p95_ms": 88.86}, "yolo11_640_fire_11class": {"avg_ms": 52.94, "count": 152, "max_ms": 77.162, "p95_ms": 72.095}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.989, "count": 152, "max_ms": 95.701, "p95_ms": 91.169}}, "truncated_count": 0}, "processing_total": {"avg_ms": 188.131, "count": 152, "max_ms": 255.042, "p95_ms": 219.397}, "read_interval": {"avg_ms": 197.403, "count": 151, "max_ms": 266.637, "p95_ms": 229.303}, "triton_wait": {"avg_ms": 66.522, "count": 152, "max_ms": 95.756, "p95_ms": 91.708}, "visualize": {"avg_ms": 8.703, "count": 152, "max_ms": 10.76, "p95_ms": 10.019}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.82, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4148.273, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:49:11+09:00", "last_error_summary": "[h264 @ 0x558cd993eac0] error while decoding MB 71 22, bytestream -13", "last_exit_at": "2026-07-01T15:49:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.399, "last_started_at": "2026-07-01T15:47:08+09:00", "pid": 6018, "process": {"exists": false, "pid": 6018}, "restart_count": 29, "restart_count_per_hour": 29, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.898, "sample_count": 1480, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15986, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:49:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T15:49:11+09:00", "last_frame_read_age_sec": 0.196, "last_frame_read_at": "2026-07-01T15:49:11+09:00", "mjpeg": {"age_sec": 0.187, "exists": true, "mtime": "2026-07-01T15:49:11.522733+09:00", "mtime_epoch": 1782888551.5227332, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196742}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:49:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.871, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 151, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.097, "pushed": 5.097, "read": 5.063}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.024, "count": 152, "max_ms": 9.392, "p95_ms": 8.087}, "gstreamer_push": {"avg_ms": 0.942, "count": 152, "max_ms": 2.863, "p95_ms": 1.788}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.59, "count": 152, "max_ms": 69.824, "p95_ms": 63.629}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.575, "count": 152, "max_ms": 95.745, "p95_ms": 88.86}, "yolo11_640_fire_11class": {"avg_ms": 52.94, "count": 152, "max_ms": 77.162, "p95_ms": 72.095}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.989, "count": 152, "max_ms": 95.701, "p95_ms": 91.169}}, "truncated_count": 0}, "processing_total": {"avg_ms": 188.131, "count": 152, "max_ms": 255.042, "p95_ms": 219.397}, "read_interval": {"avg_ms": 197.403, "count": 151, "max_ms": 266.637, "p95_ms": 229.303}, "triton_wait": {"avg_ms": 66.522, "count": 152, "max_ms": 95.756, "p95_ms": 91.708}, "visualize": {"avg_ms": 8.703, "count": 152, "max_ms": 10.76, "p95_ms": 10.019}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.823, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 133}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4157.734, "limit_mb": 8796093022207.996}, "detail": {"pid": 6277, "restart_count": 30, "start_count": 31}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:49:11+09:00", "last_error_summary": "[h264 @ 0x558cd993eac0] error while decoding MB 71 22, bytestream -13", "last_exit_at": "2026-07-01T15:49:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:49:14+09:00", "pid": 6277, "process": {"exists": true, "fd_count": 4, "hwm_mb": 44.168, "name": "ffmpeg", "pid": 6277, "rss_mb": 44.168, "state": "S (sleeping)", "threads": 1}, "restart_count": 30, "restart_count_per_hour": 30, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.498, "sample_count": 1455, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 15986, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:49:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.989, "last_frame_pushed_at": "2026-07-01T15:49:11+09:00", "last_frame_read_age_sec": 5.178, "last_frame_read_at": "2026-07-01T15:49:11+09:00", "mjpeg": {"age_sec": 4.973, "exists": true, "mtime": "2026-07-01T15:49:11.718737+09:00", "mtime_epoch": 1782888551.718737, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197071}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:49:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 870.871, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.218, "pushed": 4.218, "read": 4.184}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.057, "count": 126, "max_ms": 9.392, "p95_ms": 8.131}, "gstreamer_push": {"avg_ms": 0.937, "count": 126, "max_ms": 2.863, "p95_ms": 1.788}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.571, "count": 126, "max_ms": 69.824, "p95_ms": 64.711}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.449, "count": 126, "max_ms": 95.745, "p95_ms": 88.86}, "yolo11_640_fire_11class": {"avg_ms": 54.203, "count": 126, "max_ms": 77.162, "p95_ms": 72.443}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.627, "count": 126, "max_ms": 95.701, "p95_ms": 91.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 189.527, "count": 126, "max_ms": 255.042, "p95_ms": 221.813}, "read_interval": {"avg_ms": 198.961, "count": 125, "max_ms": 266.637, "p95_ms": 231.414}, "triton_wait": {"avg_ms": 68.242, "count": 126, "max_ms": 95.756, "p95_ms": 91.805}, "visualize": {"avg_ms": 8.68, "count": 126, "max_ms": 10.76, "p95_ms": 10.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 139}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4149.137, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 33, bytestream 40575 [h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 25, bytestream 8067 [in#0/rtsp @ 0x55b801295e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b801295e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:51:56+09:00", "last_error_summary": "[h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 33, bytestream 40575 [h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 25, bytestream 8067 [in#0/rtsp @ 0x55b801295e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b801295e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:49:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.004, "last_started_at": "2026-07-01T15:49:14+09:00", "pid": 6277, "process": {"exists": false, "pid": 6277}, "restart_count": 30, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.768, "sample_count": 1364, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16686, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:51:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T15:51:56+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T15:51:56+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T15:51:56.653729+09:00", "mtime_epoch": 1782888716.6537287, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200778}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:51:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 868.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 67, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.201, "pushed": 2.201, "read": 2.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.708, "count": 66, "max_ms": 9.697, "p95_ms": 8.469}, "gstreamer_push": {"avg_ms": 0.947, "count": 66, "max_ms": 1.749, "p95_ms": 1.101}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.735, "count": 66, "max_ms": 61.26, "p95_ms": 56.416}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.504, "count": 66, "max_ms": 87.417, "p95_ms": 68.639}, "yolo11_640_fire_11class": {"avg_ms": 51.411, "count": 66, "max_ms": 70.257, "p95_ms": 59.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.335, "count": 66, "max_ms": 85.881, "p95_ms": 72.584}}, "truncated_count": 0}, "processing_total": {"avg_ms": 172.939, "count": 66, "max_ms": 226.766, "p95_ms": 207.199}, "read_interval": {"avg_ms": 449.404, "count": 67, "max_ms": 10128.035, "p95_ms": 231.24}, "triton_wait": {"avg_ms": 62.491, "count": 66, "max_ms": 87.429, "p95_ms": 72.596}, "visualize": {"avg_ms": 8.219, "count": 66, "max_ms": 10.385, "p95_ms": 10.027}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 139}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4148.398, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:51:56+09:00", "last_error_summary": "[h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 33, bytestream 40575 [h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 25, bytestream 8067 [in#0/rtsp @ 0x55b801295e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b801295e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:51:56+09:00", "last_error_summary": "[h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 33, bytestream 40575 [h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 25, bytestream 8067 [in#0/rtsp @ 0x55b801295e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b801295e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:51:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.163, "last_started_at": "2026-07-01T15:49:14+09:00", "pid": 6277, "process": {"exists": false, "pid": 6277}, "restart_count": 30, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.768, "sample_count": 1364, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16687, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:51:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:51:56+09:00", "last_frame_read_age_sec": 0.212, "last_frame_read_at": "2026-07-01T15:51:56+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T15:51:56.653729+09:00", "mtime_epoch": 1782888716.6537287, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200778}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:51:56+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 868.25, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.234, "pushed": 2.234, "read": 2.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.73, "count": 66, "max_ms": 9.697, "p95_ms": 8.469}, "gstreamer_push": {"avg_ms": 0.946, "count": 67, "max_ms": 1.749, "p95_ms": 1.101}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.688, "count": 66, "max_ms": 61.26, "p95_ms": 56.416}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.563, "count": 66, "max_ms": 87.417, "p95_ms": 68.639}, "yolo11_640_fire_11class": {"avg_ms": 51.496, "count": 66, "max_ms": 70.257, "p95_ms": 59.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.416, "count": 66, "max_ms": 85.881, "p95_ms": 72.584}}, "truncated_count": 0}, "processing_total": {"avg_ms": 173.459, "count": 67, "max_ms": 226.766, "p95_ms": 207.791}, "read_interval": {"avg_ms": 453.619, "count": 66, "max_ms": 10128.035, "p95_ms": 231.24}, "triton_wait": {"avg_ms": 62.572, "count": 66, "max_ms": 87.429, "p95_ms": 72.596}, "visualize": {"avg_ms": 8.249, "count": 66, "max_ms": 10.385, "p95_ms": 10.027}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 139}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4148.391, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:51:56+09:00", "last_error_summary": "[h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 33, bytestream 40575 [h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 25, bytestream 8067 [in#0/rtsp @ 0x55b801295e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b801295e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:51:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.166, "last_started_at": "2026-07-01T15:49:14+09:00", "pid": 6277, "process": {"exists": false, "pid": 6277}, "restart_count": 30, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.768, "sample_count": 1364, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16687, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:51:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:51:56+09:00", "last_frame_read_age_sec": 0.215, "last_frame_read_at": "2026-07-01T15:51:56+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-01T15:51:56.653729+09:00", "mtime_epoch": 1782888716.6537287, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200778}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:51:56+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 868.25, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.234, "pushed": 2.234, "read": 2.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.73, "count": 66, "max_ms": 9.697, "p95_ms": 8.469}, "gstreamer_push": {"avg_ms": 0.946, "count": 67, "max_ms": 1.749, "p95_ms": 1.101}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.688, "count": 66, "max_ms": 61.26, "p95_ms": 56.416}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.563, "count": 66, "max_ms": 87.417, "p95_ms": 68.639}, "yolo11_640_fire_11class": {"avg_ms": 51.496, "count": 66, "max_ms": 70.257, "p95_ms": 59.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.416, "count": 66, "max_ms": 85.881, "p95_ms": 72.584}}, "truncated_count": 0}, "processing_total": {"avg_ms": 173.459, "count": 67, "max_ms": 226.766, "p95_ms": 207.791}, "read_interval": {"avg_ms": 453.619, "count": 66, "max_ms": 10128.035, "p95_ms": 231.24}, "triton_wait": {"avg_ms": 62.572, "count": 66, "max_ms": 87.429, "p95_ms": 72.596}, "visualize": {"avg_ms": 8.249, "count": 66, "max_ms": 10.385, "p95_ms": 10.027}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 139}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4253.457, "limit_mb": 8796093022207.996}, "detail": {"pid": 6384, "restart_count": 31, "start_count": 32}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:51:56+09:00", "last_error_summary": "[h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 33, bytestream 40575 [h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 25, bytestream 8067 [in#0/rtsp @ 0x55b801295e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b801295e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:51:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:51:59+09:00", "pid": 6384, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.191, "name": "ffmpeg", "pid": 6384, "rss_mb": 136.191, "state": "S (sleeping)", "threads": 106}, "restart_count": 31, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 223.768, "sample_count": 1343, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 16687, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:51:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.346, "last_frame_pushed_at": "2026-07-01T15:51:56+09:00", "last_frame_read_age_sec": 4.555, "last_frame_read_at": "2026-07-01T15:51:56+09:00", "mjpeg": {"age_sec": 4.33, "exists": true, "mtime": "2026-07-01T15:51:56.865732+09:00", "mtime_epoch": 1782888716.8657324, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:52:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 868.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 41, "pushed_frames": 41, "read_frames": 40, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.374, "pushed": 1.374, "read": 1.34}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.041, "count": 41, "max_ms": 9.697, "p95_ms": 8.575}, "gstreamer_push": {"avg_ms": 0.912, "count": 41, "max_ms": 1.101, "p95_ms": 0.986}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.533, "count": 41, "max_ms": 61.26, "p95_ms": 55.435}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.514, "count": 41, "max_ms": 87.417, "p95_ms": 76.831}, "yolo11_640_fire_11class": {"avg_ms": 52.422, "count": 41, "max_ms": 70.257, "p95_ms": 66.585}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.812, "count": 41, "max_ms": 85.881, "p95_ms": 75.761}}, "truncated_count": 0}, "processing_total": {"avg_ms": 180.719, "count": 41, "max_ms": 226.766, "p95_ms": 211.192}, "read_interval": {"avg_ms": 636.509, "count": 40, "max_ms": 10128.035, "p95_ms": 235.999}, "triton_wait": {"avg_ms": 63.978, "count": 41, "max_ms": 87.429, "p95_ms": 76.84}, "visualize": {"avg_ms": 8.463, "count": 41, "max_ms": 10.202, "p95_ms": 10.027}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.85, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 144}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4149.895, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:51:56+09:00", "last_error_summary": "[h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 33, bytestream 40575 [h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 25, bytestream 8067 [in#0/rtsp @ 0x55b801295e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b801295e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:51:56+09:00", "last_error_summary": "[h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 33, bytestream 40575 [h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 25, bytestream 8067 [in#0/rtsp @ 0x55b801295e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b801295e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:54:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.594, "last_started_at": "2026-07-01T15:51:59+09:00", "pid": 6384, "process": {"exists": false, "pid": 6384}, "restart_count": 31, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.006, "sample_count": 1351, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17301, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:54:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T15:54:04+09:00", "last_frame_read_age_sec": 0.242, "last_frame_read_at": "2026-07-01T15:54:04+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-01T15:54:04.556019+09:00", "mtime_epoch": 1782888844.5560193, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180921}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:54:04+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 868.891, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 158, "pushed_frames": 158, "read_frames": 157, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.269, "pushed": 5.269, "read": 5.235}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.842, "count": 157, "max_ms": 8.75, "p95_ms": 7.909}, "gstreamer_push": {"avg_ms": 0.939, "count": 158, "max_ms": 1.955, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.476, "count": 157, "max_ms": 70.156, "p95_ms": 63.678}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.333, "count": 157, "max_ms": 98.331, "p95_ms": 89.713}, "yolo11_640_fire_11class": {"avg_ms": 54.812, "count": 157, "max_ms": 78.27, "p95_ms": 72.949}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.659, "count": 157, "max_ms": 93.41, "p95_ms": 90.26}}, "truncated_count": 0}, "processing_total": {"avg_ms": 181.594, "count": 158, "max_ms": 235.924, "p95_ms": 218.957}, "read_interval": {"avg_ms": 190.607, "count": 157, "max_ms": 245.91, "p95_ms": 227.433}, "triton_wait": {"avg_ms": 68.379, "count": 157, "max_ms": 98.344, "p95_ms": 91.321}, "visualize": {"avg_ms": 8.243, "count": 157, "max_ms": 10.543, "p95_ms": 9.898}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 144}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4149.887, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:51:56+09:00", "last_error_summary": "[h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 33, bytestream 40575 [h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 25, bytestream 8067 [in#0/rtsp @ 0x55b801295e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b801295e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:54:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.598, "last_started_at": "2026-07-01T15:51:59+09:00", "pid": 6384, "process": {"exists": false, "pid": 6384}, "restart_count": 31, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.006, "sample_count": 1351, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17301, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:54:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T15:54:04+09:00", "last_frame_read_age_sec": 0.245, "last_frame_read_at": "2026-07-01T15:54:04+09:00", "mjpeg": {"age_sec": 0.237, "exists": true, "mtime": "2026-07-01T15:54:04.556019+09:00", "mtime_epoch": 1782888844.5560193, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180921}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:54:04+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 868.891, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 158, "pushed_frames": 158, "read_frames": 157, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.268, "pushed": 5.268, "read": 5.235}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.842, "count": 157, "max_ms": 8.75, "p95_ms": 7.909}, "gstreamer_push": {"avg_ms": 0.939, "count": 158, "max_ms": 1.955, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.476, "count": 157, "max_ms": 70.156, "p95_ms": 63.678}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.333, "count": 157, "max_ms": 98.331, "p95_ms": 89.713}, "yolo11_640_fire_11class": {"avg_ms": 54.812, "count": 157, "max_ms": 78.27, "p95_ms": 72.949}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.659, "count": 157, "max_ms": 93.41, "p95_ms": 90.26}}, "truncated_count": 0}, "processing_total": {"avg_ms": 181.594, "count": 158, "max_ms": 235.924, "p95_ms": 218.957}, "read_interval": {"avg_ms": 190.607, "count": 157, "max_ms": 245.91, "p95_ms": 227.433}, "triton_wait": {"avg_ms": 68.379, "count": 157, "max_ms": 98.344, "p95_ms": 91.321}, "visualize": {"avg_ms": 8.243, "count": 157, "max_ms": 10.543, "p95_ms": 9.898}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 144}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4261.105, "limit_mb": 8796093022207.996}, "detail": {"pid": 6491, "restart_count": 32, "start_count": 33}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:51:56+09:00", "last_error_summary": "[h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 33, bytestream 40575 [h264 @ 0x55b8012ae500] left block unavailable for requested intra mode [h264 @ 0x55b8012ae500] error while decoding MB 0 25, bytestream 8067 [in#0/rtsp @ 0x55b801295e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b801295e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:54:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:54:07+09:00", "pid": 6491, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.863, "name": "ffmpeg", "pid": 6491, "rss_mb": 141.863, "state": "S (sleeping)", "threads": 106}, "restart_count": 32, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.345, "sample_count": 1326, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17301, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:54:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.832, "last_frame_pushed_at": "2026-07-01T15:54:04+09:00", "last_frame_read_age_sec": 5.069, "last_frame_read_at": "2026-07-01T15:54:04+09:00", "mjpeg": {"age_sec": 4.817, "exists": true, "mtime": "2026-07-01T15:54:04.800024+09:00", "mtime_epoch": 1782888844.8000236, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180061}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:54:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 868.891, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.414, "pushed": 4.414, "read": 4.38}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.807, "count": 132, "max_ms": 8.75, "p95_ms": 7.897}, "gstreamer_push": {"avg_ms": 0.938, "count": 132, "max_ms": 1.955, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.988, "count": 132, "max_ms": 69.904, "p95_ms": 63.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.077, "count": 132, "max_ms": 98.331, "p95_ms": 90.494}, "yolo11_640_fire_11class": {"avg_ms": 55.342, "count": 132, "max_ms": 78.27, "p95_ms": 72.949}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.157, "count": 132, "max_ms": 93.41, "p95_ms": 90.75}}, "truncated_count": 0}, "processing_total": {"avg_ms": 181.859, "count": 132, "max_ms": 235.924, "p95_ms": 219.053}, "read_interval": {"avg_ms": 190.808, "count": 131, "max_ms": 245.91, "p95_ms": 228.307}, "triton_wait": {"avg_ms": 68.973, "count": 132, "max_ms": 98.344, "p95_ms": 91.336}, "visualize": {"avg_ms": 8.226, "count": 132, "max_ms": 10.543, "p95_ms": 9.898}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4152.816, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556ed51425c0] error while decoding MB 19 37, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:56:10+09:00", "last_error_summary": "[h264 @ 0x556ed51425c0] error while decoding MB 19 37, bytestream -9", "last_exit_at": "2026-07-01T15:54:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.167, "last_started_at": "2026-07-01T15:54:07+09:00", "pid": 6491, "process": {"exists": false, "pid": 6491}, "restart_count": 32, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.211, "sample_count": 1382, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17920, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:56:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T15:56:10+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T15:56:10+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T15:56:10.738260+09:00", "mtime_epoch": 1782888970.7382598, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194443}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:56:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 868.598, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.187, "pushed": 5.187, "read": 5.187}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.777, "count": 155, "max_ms": 9.774, "p95_ms": 8.118}, "gstreamer_push": {"avg_ms": 0.957, "count": 155, "max_ms": 2.855, "p95_ms": 1.786}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.265, "count": 155, "max_ms": 71.457, "p95_ms": 60.129}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.648, "count": 155, "max_ms": 94.189, "p95_ms": 91.849}, "yolo11_640_fire_11class": {"avg_ms": 53.245, "count": 155, "max_ms": 74.747, "p95_ms": 70.842}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.504, "count": 155, "max_ms": 92.204, "p95_ms": 90.266}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.691, "count": 155, "max_ms": 253.606, "p95_ms": 229.211}, "read_interval": {"avg_ms": 193.382, "count": 155, "max_ms": 263.921, "p95_ms": 238.618}, "triton_wait": {"avg_ms": 68.653, "count": 155, "max_ms": 94.201, "p95_ms": 91.86}, "visualize": {"avg_ms": 8.342, "count": 155, "max_ms": 12.059, "p95_ms": 9.859}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4152.09, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:56:10+09:00", "last_error_summary": "[h264 @ 0x556ed51425c0] error while decoding MB 19 37, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:56:10+09:00", "last_error_summary": "[h264 @ 0x556ed51425c0] error while decoding MB 19 37, bytestream -9", "last_exit_at": "2026-07-01T15:56:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.322, "last_started_at": "2026-07-01T15:54:07+09:00", "pid": 6491, "process": {"exists": false, "pid": 6491}, "restart_count": 32, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.211, "sample_count": 1382, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17921, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:56:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:56:10+09:00", "last_frame_read_age_sec": 0.207, "last_frame_read_at": "2026-07-01T15:56:10+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T15:56:10.738260+09:00", "mtime_epoch": 1782888970.7382598, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194443}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:56:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 868.598, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.19, "pushed": 5.19, "read": 5.157}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.793, "count": 155, "max_ms": 9.774, "p95_ms": 8.118}, "gstreamer_push": {"avg_ms": 0.956, "count": 155, "max_ms": 2.855, "p95_ms": 1.786}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.194, "count": 155, "max_ms": 71.457, "p95_ms": 60.129}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.666, "count": 155, "max_ms": 94.189, "p95_ms": 91.849}, "yolo11_640_fire_11class": {"avg_ms": 53.269, "count": 155, "max_ms": 74.747, "p95_ms": 70.842}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.533, "count": 155, "max_ms": 92.204, "p95_ms": 90.266}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.083, "count": 155, "max_ms": 253.606, "p95_ms": 229.211}, "read_interval": {"avg_ms": 193.648, "count": 154, "max_ms": 263.921, "p95_ms": 238.618}, "triton_wait": {"avg_ms": 68.675, "count": 155, "max_ms": 94.201, "p95_ms": 91.86}, "visualize": {"avg_ms": 8.362, "count": 155, "max_ms": 12.059, "p95_ms": 9.859}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.864, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4152.082, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:56:10+09:00", "last_error_summary": "[h264 @ 0x556ed51425c0] error while decoding MB 19 37, bytestream -9", "last_exit_at": "2026-07-01T15:56:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.325, "last_started_at": "2026-07-01T15:54:07+09:00", "pid": 6491, "process": {"exists": false, "pid": 6491}, "restart_count": 32, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.211, "sample_count": 1382, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17921, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:56:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T15:56:10+09:00", "last_frame_read_age_sec": 0.211, "last_frame_read_at": "2026-07-01T15:56:10+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-01T15:56:10.738260+09:00", "mtime_epoch": 1782888970.7382598, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194443}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:56:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 868.598, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.19, "pushed": 5.19, "read": 5.156}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.793, "count": 155, "max_ms": 9.774, "p95_ms": 8.118}, "gstreamer_push": {"avg_ms": 0.956, "count": 155, "max_ms": 2.855, "p95_ms": 1.786}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.194, "count": 155, "max_ms": 71.457, "p95_ms": 60.129}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.666, "count": 155, "max_ms": 94.189, "p95_ms": 91.849}, "yolo11_640_fire_11class": {"avg_ms": 53.269, "count": 155, "max_ms": 74.747, "p95_ms": 70.842}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.533, "count": 155, "max_ms": 92.204, "p95_ms": 90.266}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.083, "count": 155, "max_ms": 253.606, "p95_ms": 229.211}, "read_interval": {"avg_ms": 193.648, "count": 154, "max_ms": 263.921, "p95_ms": 238.618}, "triton_wait": {"avg_ms": 68.675, "count": 155, "max_ms": 94.201, "p95_ms": 91.86}, "visualize": {"avg_ms": 8.362, "count": 155, "max_ms": 12.059, "p95_ms": 9.859}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4257.324, "limit_mb": 8796093022207.996}, "detail": {"pid": 6598, "restart_count": 33, "start_count": 34}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:56:10+09:00", "last_error_summary": "[h264 @ 0x556ed51425c0] error while decoding MB 19 37, bytestream -9", "last_exit_at": "2026-07-01T15:56:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:56:13+09:00", "pid": 6598, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.039, "name": "ffmpeg", "pid": 6598, "rss_mb": 136.039, "state": "S (sleeping)", "threads": 106}, "restart_count": 33, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 229.325, "sample_count": 1361, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 17921, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:56:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.209, "last_frame_pushed_at": "2026-07-01T15:56:10+09:00", "last_frame_read_age_sec": 4.414, "last_frame_read_at": "2026-07-01T15:56:10+09:00", "mjpeg": {"age_sec": 4.194, "exists": true, "mtime": "2026-07-01T15:56:10.950264+09:00", "mtime_epoch": 1782888970.9502635, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195869}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:56:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 868.598, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.334, "pushed": 4.334, "read": 4.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.99, "count": 129, "max_ms": 9.774, "p95_ms": 8.239}, "gstreamer_push": {"avg_ms": 0.952, "count": 130, "max_ms": 2.855, "p95_ms": 1.792}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.577, "count": 129, "max_ms": 71.457, "p95_ms": 58.318}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.128, "count": 129, "max_ms": 94.189, "p95_ms": 91.999}, "yolo11_640_fire_11class": {"avg_ms": 54.274, "count": 129, "max_ms": 74.747, "p95_ms": 71.196}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.913, "count": 129, "max_ms": 92.204, "p95_ms": 90.774}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.048, "count": 130, "max_ms": 253.606, "p95_ms": 230.198}, "read_interval": {"avg_ms": 199.532, "count": 129, "max_ms": 263.921, "p95_ms": 239.641}, "triton_wait": {"avg_ms": 70.148, "count": 129, "max_ms": 94.201, "p95_ms": 92.01}, "visualize": {"avg_ms": 8.605, "count": 129, "max_ms": 12.059, "p95_ms": 9.874}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 154}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4151.348, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5597c33206c0] error while decoding MB 49 30, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:58:21+09:00", "last_error_summary": "[h264 @ 0x5597c33206c0] error while decoding MB 49 30, bytestream -9", "last_exit_at": "2026-07-01T15:56:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.956, "last_started_at": "2026-07-01T15:56:13+09:00", "pid": 6598, "process": {"exists": false, "pid": 6598}, "restart_count": 33, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.396, "sample_count": 1501, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18578, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:58:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T15:58:21+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T15:58:21+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T15:58:21.056558+09:00", "mtime_epoch": 1782889101.0565584, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202817}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:58:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 867.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 150, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.023, "pushed": 5.023, "read": 5.023}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.011, "count": 150, "max_ms": 10.335, "p95_ms": 7.861}, "gstreamer_push": {"avg_ms": 0.947, "count": 150, "max_ms": 1.963, "p95_ms": 1.775}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.34, "count": 150, "max_ms": 70.656, "p95_ms": 65.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.211, "count": 150, "max_ms": 94.293, "p95_ms": 89.911}, "yolo11_640_fire_11class": {"avg_ms": 55.038, "count": 150, "max_ms": 80.784, "p95_ms": 71.474}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.055, "count": 150, "max_ms": 94.129, "p95_ms": 90.912}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.197, "count": 150, "max_ms": 251.518, "p95_ms": 229.885}, "read_interval": {"avg_ms": 200.026, "count": 150, "max_ms": 263.46, "p95_ms": 240.865}, "triton_wait": {"avg_ms": 68.497, "count": 150, "max_ms": 94.306, "p95_ms": 90.925}, "visualize": {"avg_ms": 8.694, "count": 150, "max_ms": 12.289, "p95_ms": 10.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.864, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 154}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4150.492, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T15:58:21+09:00", "last_error_summary": "[h264 @ 0x5597c33206c0] error while decoding MB 49 30, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:58:21+09:00", "last_error_summary": "[h264 @ 0x5597c33206c0] error while decoding MB 49 30, bytestream -9", "last_exit_at": "2026-07-01T15:58:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.125, "last_started_at": "2026-07-01T15:56:13+09:00", "pid": 6598, "process": {"exists": false, "pid": 6598}, "restart_count": 33, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.396, "sample_count": 1501, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18579, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:58:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T15:58:21+09:00", "last_frame_read_age_sec": 0.221, "last_frame_read_at": "2026-07-01T15:58:21+09:00", "mjpeg": {"age_sec": 0.212, "exists": true, "mtime": "2026-07-01T15:58:21.056558+09:00", "mtime_epoch": 1782889101.0565584, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202817}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:58:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 867.07, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.029, "pushed": 5.029, "read": 4.995}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.017, "count": 150, "max_ms": 10.335, "p95_ms": 7.861}, "gstreamer_push": {"avg_ms": 0.947, "count": 150, "max_ms": 1.963, "p95_ms": 1.775}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.244, "count": 150, "max_ms": 70.656, "p95_ms": 65.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.231, "count": 150, "max_ms": 94.293, "p95_ms": 90.085}, "yolo11_640_fire_11class": {"avg_ms": 55.016, "count": 150, "max_ms": 80.784, "p95_ms": 71.472}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.076, "count": 150, "max_ms": 94.129, "p95_ms": 91.332}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.364, "count": 150, "max_ms": 251.518, "p95_ms": 229.885}, "read_interval": {"avg_ms": 200.019, "count": 149, "max_ms": 263.46, "p95_ms": 240.865}, "triton_wait": {"avg_ms": 68.519, "count": 150, "max_ms": 94.306, "p95_ms": 91.345}, "visualize": {"avg_ms": 8.701, "count": 150, "max_ms": 12.289, "p95_ms": 10.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.829, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 154}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4150.484, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:58:21+09:00", "last_error_summary": "[h264 @ 0x5597c33206c0] error while decoding MB 49 30, bytestream -9", "last_exit_at": "2026-07-01T15:58:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.129, "last_started_at": "2026-07-01T15:56:13+09:00", "pid": 6598, "process": {"exists": false, "pid": 6598}, "restart_count": 33, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.396, "sample_count": 1501, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18579, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:58:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T15:58:21+09:00", "last_frame_read_age_sec": 0.225, "last_frame_read_at": "2026-07-01T15:58:21+09:00", "mjpeg": {"age_sec": 0.216, "exists": true, "mtime": "2026-07-01T15:58:21.056558+09:00", "mtime_epoch": 1782889101.0565584, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202817}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:58:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 867.07, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.028, "pushed": 5.028, "read": 4.994}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.017, "count": 150, "max_ms": 10.335, "p95_ms": 7.861}, "gstreamer_push": {"avg_ms": 0.947, "count": 150, "max_ms": 1.963, "p95_ms": 1.775}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.244, "count": 150, "max_ms": 70.656, "p95_ms": 65.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.231, "count": 150, "max_ms": 94.293, "p95_ms": 90.085}, "yolo11_640_fire_11class": {"avg_ms": 55.016, "count": 150, "max_ms": 80.784, "p95_ms": 71.472}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.076, "count": 150, "max_ms": 94.129, "p95_ms": 91.332}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.364, "count": 150, "max_ms": 251.518, "p95_ms": 229.885}, "read_interval": {"avg_ms": 200.019, "count": 149, "max_ms": 263.46, "p95_ms": 240.865}, "triton_wait": {"avg_ms": 68.519, "count": 150, "max_ms": 94.306, "p95_ms": 91.345}, "visualize": {"avg_ms": 8.701, "count": 150, "max_ms": 12.289, "p95_ms": 10.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.833, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 154}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4269.051, "limit_mb": 8796093022207.996}, "detail": {"pid": 6705, "restart_count": 34, "start_count": 35}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T15:58:21+09:00", "last_error_summary": "[h264 @ 0x5597c33206c0] error while decoding MB 49 30, bytestream -9", "last_exit_at": "2026-07-01T15:58:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T15:58:23+09:00", "pid": 6705, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.719, "name": "ffmpeg", "pid": 6705, "rss_mb": 149.719, "state": "S (sleeping)", "threads": 106}, "restart_count": 34, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.396, "sample_count": 1480, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 18579, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T15:58:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.389, "last_frame_pushed_at": "2026-07-01T15:58:21+09:00", "last_frame_read_age_sec": 4.607, "last_frame_read_at": "2026-07-01T15:58:21+09:00", "mjpeg": {"age_sec": 4.374, "exists": true, "mtime": "2026-07-01T15:58:21.280562+09:00", "mtime_epoch": 1782889101.2805622, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202764}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T15:58:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.449, "name": "celery", "pid": 237, "rss_mb": 867.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.292, "pushed": 4.292, "read": 4.258}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.061, "count": 128, "max_ms": 10.335, "p95_ms": 7.861}, "gstreamer_push": {"avg_ms": 0.942, "count": 128, "max_ms": 1.952, "p95_ms": 1.775}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.306, "count": 128, "max_ms": 70.656, "p95_ms": 63.516}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.263, "count": 128, "max_ms": 94.293, "p95_ms": 89.911}, "yolo11_640_fire_11class": {"avg_ms": 54.321, "count": 128, "max_ms": 80.784, "p95_ms": 71.474}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.205, "count": 128, "max_ms": 94.129, "p95_ms": 91.332}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.483, "count": 128, "max_ms": 251.518, "p95_ms": 230.901}, "read_interval": {"avg_ms": 200.043, "count": 127, "max_ms": 263.46, "p95_ms": 241.191}, "triton_wait": {"avg_ms": 67.592, "count": 128, "max_ms": 94.306, "p95_ms": 91.345}, "visualize": {"avg_ms": 8.754, "count": 128, "max_ms": 12.289, "p95_ms": 10.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.825, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 160}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4160.641, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560a6f3d6780] error while decoding MB 41 15, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:00:27+09:00", "last_error_summary": "[h264 @ 0x560a6f3d6780] error while decoding MB 41 15, bytestream -7", "last_exit_at": "2026-07-01T15:58:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.821, "last_started_at": "2026-07-01T15:58:23+09:00", "pid": 6705, "process": {"exists": false, "pid": 6705}, "restart_count": 34, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.849, "sample_count": 1517, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19212, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:00:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T16:00:27+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T16:00:27+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T16:00:27.434776+09:00", "mtime_epoch": 1782889227.4347756, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198833}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:00:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 875.789, "name": "celery", "pid": 237, "rss_mb": 875.074, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.095, "pushed": 5.095, "read": 5.095}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.057, "count": 152, "max_ms": 10.98, "p95_ms": 8.638}, "gstreamer_push": {"avg_ms": 0.974, "count": 152, "max_ms": 2.923, "p95_ms": 1.791}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.649, "count": 152, "max_ms": 69.503, "p95_ms": 65.02}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.084, "count": 152, "max_ms": 93.867, "p95_ms": 89.038}, "yolo11_640_fire_11class": {"avg_ms": 54.593, "count": 152, "max_ms": 77.539, "p95_ms": 70.402}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.944, "count": 152, "max_ms": 93.282, "p95_ms": 89.822}}, "truncated_count": 0}, "processing_total": {"avg_ms": 187.436, "count": 152, "max_ms": 275.367, "p95_ms": 231.717}, "read_interval": {"avg_ms": 197.237, "count": 152, "max_ms": 289.737, "p95_ms": 241.795}, "triton_wait": {"avg_ms": 68.42, "count": 152, "max_ms": 93.88, "p95_ms": 90.008}, "visualize": {"avg_ms": 8.487, "count": 152, "max_ms": 13.081, "p95_ms": 10.087}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.836, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 160}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4159.898, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:00:27+09:00", "last_error_summary": "[h264 @ 0x560a6f3d6780] error while decoding MB 41 15, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:00:27+09:00", "last_error_summary": "[h264 @ 0x560a6f3d6780] error while decoding MB 41 15, bytestream -7", "last_exit_at": "2026-07-01T16:00:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.96, "last_started_at": "2026-07-01T15:58:23+09:00", "pid": 6705, "process": {"exists": false, "pid": 6705}, "restart_count": 34, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.849, "sample_count": 1517, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19213, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:00:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:00:27+09:00", "last_frame_read_age_sec": 0.19, "last_frame_read_at": "2026-07-01T16:00:27+09:00", "mjpeg": {"age_sec": 0.179, "exists": true, "mtime": "2026-07-01T16:00:27.434776+09:00", "mtime_epoch": 1782889227.4347756, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198833}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:00:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 875.789, "name": "celery", "pid": 237, "rss_mb": 875.074, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.104, "pushed": 5.104, "read": 5.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.054, "count": 152, "max_ms": 10.98, "p95_ms": 8.638}, "gstreamer_push": {"avg_ms": 0.973, "count": 153, "max_ms": 2.923, "p95_ms": 1.791}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.621, "count": 152, "max_ms": 69.503, "p95_ms": 65.02}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.105, "count": 152, "max_ms": 93.867, "p95_ms": 89.038}, "yolo11_640_fire_11class": {"avg_ms": 54.623, "count": 152, "max_ms": 77.539, "p95_ms": 70.402}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.979, "count": 152, "max_ms": 93.282, "p95_ms": 89.822}}, "truncated_count": 0}, "processing_total": {"avg_ms": 187.426, "count": 153, "max_ms": 275.367, "p95_ms": 231.717}, "read_interval": {"avg_ms": 197.237, "count": 152, "max_ms": 289.737, "p95_ms": 241.795}, "triton_wait": {"avg_ms": 68.449, "count": 152, "max_ms": 93.88, "p95_ms": 90.008}, "visualize": {"avg_ms": 8.489, "count": 152, "max_ms": 13.081, "p95_ms": 10.087}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 160}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4159.891, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:00:27+09:00", "last_error_summary": "[h264 @ 0x560a6f3d6780] error while decoding MB 41 15, bytestream -7", "last_exit_at": "2026-07-01T16:00:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.964, "last_started_at": "2026-07-01T15:58:23+09:00", "pid": 6705, "process": {"exists": false, "pid": 6705}, "restart_count": 34, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.849, "sample_count": 1517, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19213, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:00:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:00:27+09:00", "last_frame_read_age_sec": 0.194, "last_frame_read_at": "2026-07-01T16:00:27+09:00", "mjpeg": {"age_sec": 0.182, "exists": true, "mtime": "2026-07-01T16:00:27.434776+09:00", "mtime_epoch": 1782889227.4347756, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198833}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:00:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 875.789, "name": "celery", "pid": 237, "rss_mb": 875.074, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 153, "pushed_frames": 153, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.104, "pushed": 5.104, "read": 5.07}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.054, "count": 152, "max_ms": 10.98, "p95_ms": 8.638}, "gstreamer_push": {"avg_ms": 0.973, "count": 153, "max_ms": 2.923, "p95_ms": 1.791}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.621, "count": 152, "max_ms": 69.503, "p95_ms": 65.02}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.105, "count": 152, "max_ms": 93.867, "p95_ms": 89.038}, "yolo11_640_fire_11class": {"avg_ms": 54.623, "count": 152, "max_ms": 77.539, "p95_ms": 70.402}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.979, "count": 152, "max_ms": 93.282, "p95_ms": 89.822}}, "truncated_count": 0}, "processing_total": {"avg_ms": 187.426, "count": 153, "max_ms": 275.367, "p95_ms": 231.717}, "read_interval": {"avg_ms": 197.237, "count": 152, "max_ms": 289.737, "p95_ms": 241.795}, "triton_wait": {"avg_ms": 68.449, "count": 152, "max_ms": 93.88, "p95_ms": 90.008}, "visualize": {"avg_ms": 8.489, "count": 152, "max_ms": 13.081, "p95_ms": 10.087}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 160}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4268.332, "limit_mb": 8796093022207.996}, "detail": {"pid": 6812, "restart_count": 35, "start_count": 36}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:00:27+09:00", "last_error_summary": "[h264 @ 0x560a6f3d6780] error while decoding MB 41 15, bytestream -7", "last_exit_at": "2026-07-01T16:00:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:00:30+09:00", "pid": 6812, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.246, "name": "ffmpeg", "pid": 6812, "rss_mb": 139.246, "state": "S (sleeping)", "threads": 106}, "restart_count": 35, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 220.849, "sample_count": 1493, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19213, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:00:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.489, "last_frame_pushed_at": "2026-07-01T16:00:27+09:00", "last_frame_read_age_sec": 4.676, "last_frame_read_at": "2026-07-01T16:00:27+09:00", "mjpeg": {"age_sec": 4.473, "exists": true, "mtime": "2026-07-01T16:00:27.626779+09:00", "mtime_epoch": 1782889227.6267788, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192195}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:00:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 875.789, "name": "celery", "pid": 237, "rss_mb": 875.074, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.347, "pushed": 4.347, "read": 4.314}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.007, "count": 130, "max_ms": 10.98, "p95_ms": 9.083}, "gstreamer_push": {"avg_ms": 0.975, "count": 130, "max_ms": 2.923, "p95_ms": 1.144}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.631, "count": 130, "max_ms": 69.503, "p95_ms": 61.413}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.959, "count": 130, "max_ms": 92.585, "p95_ms": 88.11}, "yolo11_640_fire_11class": {"avg_ms": 54.585, "count": 130, "max_ms": 77.539, "p95_ms": 70.26}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.046, "count": 130, "max_ms": 93.282, "p95_ms": 88.389}}, "truncated_count": 0}, "processing_total": {"avg_ms": 186.984, "count": 130, "max_ms": 275.367, "p95_ms": 232.586}, "read_interval": {"avg_ms": 196.775, "count": 129, "max_ms": 289.737, "p95_ms": 243.353}, "triton_wait": {"avg_ms": 68.501, "count": 130, "max_ms": 93.293, "p95_ms": 89.72}, "visualize": {"avg_ms": 8.474, "count": 130, "max_ms": 13.081, "p95_ms": 10.087}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 165}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4166.738, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5572dc913ac0] error while decoding MB 62 37, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:02:38+09:00", "last_error_summary": "[h264 @ 0x5572dc913ac0] error while decoding MB 62 37, bytestream -9", "last_exit_at": "2026-07-01T16:00:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.958, "last_started_at": "2026-07-01T16:00:30+09:00", "pid": 6812, "process": {"exists": false, "pid": 6812}, "restart_count": 35, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.803, "sample_count": 1520, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19878, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:02:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T16:02:37+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T16:02:38+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T16:02:38.017057+09:00", "mtime_epoch": 1782889358.017057, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197319}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:02:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 870.328, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 150, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.019, "pushed": 5.019, "read": 5.019}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.019, "count": 149, "max_ms": 9.061, "p95_ms": 7.987}, "gstreamer_push": {"avg_ms": 0.962, "count": 150, "max_ms": 1.97, "p95_ms": 1.838}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.042, "count": 149, "max_ms": 72.781, "p95_ms": 67.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.135, "count": 149, "max_ms": 92.825, "p95_ms": 89.803}, "yolo11_640_fire_11class": {"avg_ms": 53.828, "count": 149, "max_ms": 71.857, "p95_ms": 70.18}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.364, "count": 149, "max_ms": 94.157, "p95_ms": 91.025}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.118, "count": 150, "max_ms": 257.293, "p95_ms": 230.377}, "read_interval": {"avg_ms": 200.201, "count": 150, "max_ms": 269.231, "p95_ms": 240.756}, "triton_wait": {"avg_ms": 68.793, "count": 149, "max_ms": 94.171, "p95_ms": 91.626}, "visualize": {"avg_ms": 8.689, "count": 149, "max_ms": 11.393, "p95_ms": 9.997}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 165}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4165.828, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:02:38+09:00", "last_error_summary": "[h264 @ 0x5572dc913ac0] error while decoding MB 62 37, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:02:38+09:00", "last_error_summary": "[h264 @ 0x5572dc913ac0] error while decoding MB 62 37, bytestream -9", "last_exit_at": "2026-07-01T16:02:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.095, "last_started_at": "2026-07-01T16:00:30+09:00", "pid": 6812, "process": {"exists": false, "pid": 6812}, "restart_count": 35, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.803, "sample_count": 1520, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19879, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:02:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:02:38+09:00", "last_frame_read_age_sec": 0.188, "last_frame_read_at": "2026-07-01T16:02:38+09:00", "mjpeg": {"age_sec": 0.176, "exists": true, "mtime": "2026-07-01T16:02:38.017057+09:00", "mtime_epoch": 1782889358.017057, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197319}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:02:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 870.328, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.032, "pushed": 5.032, "read": 4.998}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.048, "count": 150, "max_ms": 11.264, "p95_ms": 7.998}, "gstreamer_push": {"avg_ms": 0.962, "count": 150, "max_ms": 1.97, "p95_ms": 1.838}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.982, "count": 150, "max_ms": 72.781, "p95_ms": 67.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.077, "count": 150, "max_ms": 92.825, "p95_ms": 89.803}, "yolo11_640_fire_11class": {"avg_ms": 53.764, "count": 150, "max_ms": 71.857, "p95_ms": 70.18}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.284, "count": 150, "max_ms": 94.157, "p95_ms": 91.025}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.022, "count": 150, "max_ms": 257.293, "p95_ms": 230.377}, "read_interval": {"avg_ms": 200.15, "count": 149, "max_ms": 269.231, "p95_ms": 240.756}, "triton_wait": {"avg_ms": 68.718, "count": 150, "max_ms": 94.171, "p95_ms": 91.626}, "visualize": {"avg_ms": 8.692, "count": 150, "max_ms": 11.393, "p95_ms": 9.997}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.812, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 165}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4165.941, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:02:38+09:00", "last_error_summary": "[h264 @ 0x5572dc913ac0] error while decoding MB 62 37, bytestream -9", "last_exit_at": "2026-07-01T16:02:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.098, "last_started_at": "2026-07-01T16:00:30+09:00", "pid": 6812, "process": {"exists": false, "pid": 6812}, "restart_count": 35, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.803, "sample_count": 1520, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19879, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:02:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T16:02:38+09:00", "last_frame_read_age_sec": 0.192, "last_frame_read_at": "2026-07-01T16:02:38+09:00", "mjpeg": {"age_sec": 0.18, "exists": true, "mtime": "2026-07-01T16:02:38.017057+09:00", "mtime_epoch": 1782889358.017057, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197319}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:02:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 870.328, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.031, "pushed": 5.031, "read": 4.997}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.048, "count": 150, "max_ms": 11.264, "p95_ms": 7.998}, "gstreamer_push": {"avg_ms": 0.962, "count": 150, "max_ms": 1.97, "p95_ms": 1.838}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.982, "count": 150, "max_ms": 72.781, "p95_ms": 67.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.077, "count": 150, "max_ms": 92.825, "p95_ms": 89.803}, "yolo11_640_fire_11class": {"avg_ms": 53.764, "count": 150, "max_ms": 71.857, "p95_ms": 70.18}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.284, "count": 150, "max_ms": 94.157, "p95_ms": 91.025}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.022, "count": 150, "max_ms": 257.293, "p95_ms": 230.377}, "read_interval": {"avg_ms": 200.15, "count": 149, "max_ms": 269.231, "p95_ms": 240.756}, "triton_wait": {"avg_ms": 68.718, "count": 150, "max_ms": 94.171, "p95_ms": 91.626}, "visualize": {"avg_ms": 8.692, "count": 150, "max_ms": 11.393, "p95_ms": 9.997}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.815, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 165}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4271.52, "limit_mb": 8796093022207.996}, "detail": {"pid": 6979, "restart_count": 36, "start_count": 37}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:02:38+09:00", "last_error_summary": "[h264 @ 0x5572dc913ac0] error while decoding MB 62 37, bytestream -9", "last_exit_at": "2026-07-01T16:02:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:02:40+09:00", "pid": 6979, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.871, "name": "ffmpeg", "pid": 6979, "rss_mb": 136.871, "state": "S (sleeping)", "threads": 106}, "restart_count": 36, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 219.091, "sample_count": 1496, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19879, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:02:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.337, "last_frame_pushed_at": "2026-07-01T16:02:38+09:00", "last_frame_read_age_sec": 4.522, "last_frame_read_at": "2026-07-01T16:02:38+09:00", "mjpeg": {"age_sec": 4.323, "exists": true, "mtime": "2026-07-01T16:02:38.205060+09:00", "mtime_epoch": 1782889358.2050602, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197584}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:02:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 870.328, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.324, "pushed": 4.324, "read": 4.29}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.047, "count": 129, "max_ms": 11.264, "p95_ms": 7.987}, "gstreamer_push": {"avg_ms": 0.961, "count": 129, "max_ms": 1.97, "p95_ms": 1.83}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.25, "count": 129, "max_ms": 72.781, "p95_ms": 67.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.306, "count": 129, "max_ms": 92.825, "p95_ms": 89.803}, "yolo11_640_fire_11class": {"avg_ms": 53.952, "count": 129, "max_ms": 71.857, "p95_ms": 70.18}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.365, "count": 129, "max_ms": 94.157, "p95_ms": 89.953}}, "truncated_count": 0}, "processing_total": {"avg_ms": 189.495, "count": 129, "max_ms": 257.293, "p95_ms": 224.991}, "read_interval": {"avg_ms": 199.63, "count": 128, "max_ms": 269.231, "p95_ms": 234.754}, "triton_wait": {"avg_ms": 68.842, "count": 129, "max_ms": 94.171, "p95_ms": 91.501}, "visualize": {"avg_ms": 8.663, "count": 129, "max_ms": 11.393, "p95_ms": 9.971}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.837, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 356}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4098.723, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556adacfce00] error while decoding MB 95 28, bytestream -21 [in#0/rtsp @ 0x556adab55e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556adab55e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:03:45+09:00", "last_error_summary": "[h264 @ 0x556adacfce00] error while decoding MB 95 28, bytestream -21 [in#0/rtsp @ 0x556adab55e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556adab55e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T15:48:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 896.295, "last_started_at": "2026-07-01T15:48:49+09:00", "pid": 6125, "process": {"exists": false, "pid": 6125}, "restart_count": 4, "restart_count_per_hour": 2, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.559, "sample_count": 2513, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42813, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:03:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-01T16:03:45+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T16:03:45+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-01T16:03:45.766216+09:00", "mtime_epoch": 1782889425.7662156, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 265504}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:03:45+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 1110.527, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 19, "pushed_frames": 19, "read_frames": 19, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.634, "pushed": 0.634, "read": 0.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.435, "count": 19, "max_ms": 4.455, "p95_ms": 4.455}, "gstreamer_push": {"avg_ms": 1.184, "count": 19, "max_ms": 1.439, "p95_ms": 1.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.369, "count": 19, "max_ms": 57.558, "p95_ms": 57.558}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.054, "count": 19, "max_ms": 92.046, "p95_ms": 92.046}, "yolo11_640_fire_11class": {"avg_ms": 55.436, "count": 19, "max_ms": 75.132, "p95_ms": 75.132}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.977, "count": 19, "max_ms": 89.433, "p95_ms": 89.433}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.304, "count": 19, "max_ms": 144.939, "p95_ms": 144.939}, "read_interval": {"avg_ms": 1580.487, "count": 19, "max_ms": 27963.305, "p95_ms": 27963.305}, "triton_wait": {"avg_ms": 64.066, "count": 19, "max_ms": 92.058, "p95_ms": 92.058}, "visualize": {"avg_ms": 4.527, "count": 19, "max_ms": 5.142, "p95_ms": 5.142}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 356}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4097.973, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:03:45+09:00", "last_error_summary": "[h264 @ 0x556adacfce00] error while decoding MB 95 28, bytestream -21 [in#0/rtsp @ 0x556adab55e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556adab55e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:03:45+09:00", "last_error_summary": "[h264 @ 0x556adacfce00] error while decoding MB 95 28, bytestream -21 [in#0/rtsp @ 0x556adab55e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556adab55e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T16:03:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 896.351, "last_started_at": "2026-07-01T15:48:49+09:00", "pid": 6125, "process": {"exists": false, "pid": 6125}, "restart_count": 4, "restart_count_per_hour": 2, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.559, "sample_count": 2514, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42814, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:03:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T16:03:45+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-01T16:03:45+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T16:03:45.766216+09:00", "mtime_epoch": 1782889425.7662156, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 265504}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:03:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 1110.527, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 19, "pushed_frames": 19, "read_frames": 19, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.633, "pushed": 0.633, "read": 0.633}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.479, "count": 19, "max_ms": 4.976, "p95_ms": 4.976}, "gstreamer_push": {"avg_ms": 1.188, "count": 19, "max_ms": 1.439, "p95_ms": 1.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.713, "count": 19, "max_ms": 57.558, "p95_ms": 57.558}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.313, "count": 19, "max_ms": 89.42, "p95_ms": 89.42}, "yolo11_640_fire_11class": {"avg_ms": 54.256, "count": 19, "max_ms": 71.099, "p95_ms": 71.099}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.795, "count": 19, "max_ms": 89.433, "p95_ms": 89.433}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.906, "count": 19, "max_ms": 144.939, "p95_ms": 144.939}, "read_interval": {"avg_ms": 1580.487, "count": 19, "max_ms": 27963.305, "p95_ms": 27963.305}, "triton_wait": {"avg_ms": 62.326, "count": 19, "max_ms": 89.443, "p95_ms": 89.443}, "visualize": {"avg_ms": 4.755, "count": 19, "max_ms": 9.468, "p95_ms": 9.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 356}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4097.965, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:03:45+09:00", "last_error_summary": "[h264 @ 0x556adacfce00] error while decoding MB 95 28, bytestream -21 [in#0/rtsp @ 0x556adab55e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556adab55e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T16:03:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 896.357, "last_started_at": "2026-07-01T15:48:49+09:00", "pid": 6125, "process": {"exists": false, "pid": 6125}, "restart_count": 4, "restart_count_per_hour": 2, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.559, "sample_count": 2514, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42814, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:03:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T16:03:45+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-01T16:03:45+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-01T16:03:45.766216+09:00", "mtime_epoch": 1782889425.7662156, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 265504}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:03:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 1110.527, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 19, "pushed_frames": 19, "read_frames": 18, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.636, "pushed": 0.636, "read": 0.602}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.479, "count": 19, "max_ms": 4.976, "p95_ms": 4.976}, "gstreamer_push": {"avg_ms": 1.188, "count": 19, "max_ms": 1.439, "p95_ms": 1.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.713, "count": 19, "max_ms": 57.558, "p95_ms": 57.558}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.313, "count": 19, "max_ms": 89.42, "p95_ms": 89.42}, "yolo11_640_fire_11class": {"avg_ms": 54.256, "count": 19, "max_ms": 71.099, "p95_ms": 71.099}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.795, "count": 19, "max_ms": 89.433, "p95_ms": 89.433}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.906, "count": 19, "max_ms": 144.939, "p95_ms": 144.939}, "read_interval": {"avg_ms": 1659.84, "count": 18, "max_ms": 27963.305, "p95_ms": 27963.305}, "triton_wait": {"avg_ms": 62.326, "count": 19, "max_ms": 89.443, "p95_ms": 89.443}, "visualize": {"avg_ms": 4.755, "count": 19, "max_ms": 9.468, "p95_ms": 9.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 356}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4130.129, "limit_mb": 8796093022207.996}, "detail": {"pid": 7086, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:03:45+09:00", "last_error_summary": "[h264 @ 0x556adacfce00] error while decoding MB 95 28, bytestream -21 [in#0/rtsp @ 0x556adab55e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556adab55e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T16:03:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:03:48+09:00", "pid": 7086, "process": {"exists": true, "fd_count": 5, "hwm_mb": 68.523, "name": "ffmpeg", "pid": 7086, "rss_mb": 68.523, "state": "S (sleeping)", "threads": 19}, "restart_count": 5, "restart_count_per_hour": 3, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.663, "sample_count": 2472, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42814, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:03:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.515, "last_frame_pushed_at": "2026-07-01T16:03:45+09:00", "last_frame_read_age_sec": 4.63, "last_frame_read_at": "2026-07-01T16:03:45+09:00", "mjpeg": {"age_sec": 4.498, "exists": true, "mtime": "2026-07-01T16:03:45.886218+09:00", "mtime_epoch": 1782889425.8862176, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259025}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:03:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 1110.527, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.172, "pushed": 2.172, "read": 2.172}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.669, "count": 13, "max_ms": 4.976, "p95_ms": 4.976}, "gstreamer_push": {"avg_ms": 1.189, "count": 13, "max_ms": 1.439, "p95_ms": 1.439}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.679, "count": 13, "max_ms": 57.558, "p95_ms": 57.558}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.136, "count": 13, "max_ms": 89.42, "p95_ms": 89.42}, "yolo11_640_fire_11class": {"avg_ms": 54.055, "count": 13, "max_ms": 71.099, "p95_ms": 71.099}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.265, "count": 13, "max_ms": 89.433, "p95_ms": 89.433}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.627, "count": 13, "max_ms": 144.939, "p95_ms": 144.939}, "read_interval": {"avg_ms": 2255.307, "count": 13, "max_ms": 27963.305, "p95_ms": 27963.305}, "triton_wait": {"avg_ms": 62.149, "count": 13, "max_ms": 89.443, "p95_ms": 89.443}, "visualize": {"avg_ms": 4.922, "count": 13, "max_ms": 9.468, "p95_ms": 9.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 170}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4173.336, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56178328f400] error while decoding MB 17 6, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:04:49+09:00", "last_error_summary": "[h264 @ 0x56178328f400] error while decoding MB 17 6, bytestream -9", "last_exit_at": "2026-07-01T16:02:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.454, "last_started_at": "2026-07-01T16:02:40+09:00", "pid": 6979, "process": {"exists": false, "pid": 6979}, "restart_count": 36, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.126, "sample_count": 1496, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20515, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:04:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T16:04:49+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T16:04:49+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T16:04:49.939263+09:00", "mtime_epoch": 1782889489.9392626, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206324}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:04:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 872.812, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.979, "pushed": 4.979, "read": 4.979}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.196, "count": 148, "max_ms": 9.845, "p95_ms": 7.974}, "gstreamer_push": {"avg_ms": 0.947, "count": 149, "max_ms": 2.135, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.588, "count": 148, "max_ms": 72.178, "p95_ms": 68.391}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.297, "count": 148, "max_ms": 93.466, "p95_ms": 89.412}, "yolo11_640_fire_11class": {"avg_ms": 54.074, "count": 148, "max_ms": 74.894, "p95_ms": 70.294}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.668, "count": 148, "max_ms": 94.652, "p95_ms": 90.41}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.106, "count": 149, "max_ms": 262.033, "p95_ms": 229.154}, "read_interval": {"avg_ms": 201.71, "count": 149, "max_ms": 272.335, "p95_ms": 238.554}, "triton_wait": {"avg_ms": 68.273, "count": 148, "max_ms": 94.662, "p95_ms": 91.157}, "visualize": {"avg_ms": 8.745, "count": 148, "max_ms": 11.43, "p95_ms": 10.047}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 170}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4171.879, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:04:49+09:00", "last_error_summary": "[h264 @ 0x56178328f400] error while decoding MB 17 6, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:04:49+09:00", "last_error_summary": "[h264 @ 0x56178328f400] error while decoding MB 17 6, bytestream -9", "last_exit_at": "2026-07-01T16:04:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.555, "last_started_at": "2026-07-01T16:02:40+09:00", "pid": 6979, "process": {"exists": false, "pid": 6979}, "restart_count": 36, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.126, "sample_count": 1496, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20516, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:04:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:04:50+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-01T16:04:49+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-01T16:04:49.939263+09:00", "mtime_epoch": 1782889489.9392626, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206324}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:04:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 872.812, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.994, "pushed": 4.994, "read": 4.961}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.189, "count": 149, "max_ms": 9.845, "p95_ms": 7.974}, "gstreamer_push": {"avg_ms": 0.948, "count": 149, "max_ms": 2.135, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.653, "count": 149, "max_ms": 72.178, "p95_ms": 68.391}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.301, "count": 149, "max_ms": 93.466, "p95_ms": 89.412}, "yolo11_640_fire_11class": {"avg_ms": 54.095, "count": 149, "max_ms": 74.894, "p95_ms": 70.294}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.666, "count": 149, "max_ms": 94.652, "p95_ms": 90.41}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.905, "count": 149, "max_ms": 262.033, "p95_ms": 229.154}, "read_interval": {"avg_ms": 201.825, "count": 148, "max_ms": 272.335, "p95_ms": 238.554}, "triton_wait": {"avg_ms": 68.268, "count": 149, "max_ms": 94.662, "p95_ms": 91.157}, "visualize": {"avg_ms": 8.729, "count": 149, "max_ms": 11.43, "p95_ms": 10.047}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.834, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 170}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4171.871, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:04:49+09:00", "last_error_summary": "[h264 @ 0x56178328f400] error while decoding MB 17 6, bytestream -9", "last_exit_at": "2026-07-01T16:04:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.559, "last_started_at": "2026-07-01T16:02:40+09:00", "pid": 6979, "process": {"exists": false, "pid": 6979}, "restart_count": 36, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.126, "sample_count": 1496, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20516, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:04:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T16:04:50+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-01T16:04:49+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-01T16:04:49.939263+09:00", "mtime_epoch": 1782889489.9392626, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206324}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:04:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 872.812, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.994, "pushed": 4.994, "read": 4.96}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.189, "count": 149, "max_ms": 9.845, "p95_ms": 7.974}, "gstreamer_push": {"avg_ms": 0.948, "count": 149, "max_ms": 2.135, "p95_ms": 1.314}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.653, "count": 149, "max_ms": 72.178, "p95_ms": 68.391}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.301, "count": 149, "max_ms": 93.466, "p95_ms": 89.412}, "yolo11_640_fire_11class": {"avg_ms": 54.095, "count": 149, "max_ms": 74.894, "p95_ms": 70.294}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.666, "count": 149, "max_ms": 94.652, "p95_ms": 90.41}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.905, "count": 149, "max_ms": 262.033, "p95_ms": 229.154}, "read_interval": {"avg_ms": 201.825, "count": 148, "max_ms": 272.335, "p95_ms": 238.554}, "triton_wait": {"avg_ms": 68.268, "count": 149, "max_ms": 94.662, "p95_ms": 91.157}, "visualize": {"avg_ms": 8.729, "count": 149, "max_ms": 11.43, "p95_ms": 10.047}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.838, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 170}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4280.617, "limit_mb": 8796093022207.996}, "detail": {"pid": 7298, "restart_count": 37, "start_count": 38}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:04:49+09:00", "last_error_summary": "[h264 @ 0x56178328f400] error while decoding MB 17 6, bytestream -9", "last_exit_at": "2026-07-01T16:04:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:04:52+09:00", "pid": 7298, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.008, "name": "ffmpeg", "pid": 7298, "rss_mb": 139.008, "state": "S (sleeping)", "threads": 106}, "restart_count": 37, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 222.126, "sample_count": 1471, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20516, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:04:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.544, "last_frame_pushed_at": "2026-07-01T16:04:50+09:00", "last_frame_read_age_sec": 4.692, "last_frame_read_at": "2026-07-01T16:04:49+09:00", "mjpeg": {"age_sec": 4.524, "exists": true, "mtime": "2026-07-01T16:04:50.099265+09:00", "mtime_epoch": 1782889490.099265, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195304}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:04:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 872.812, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.176, "pushed": 4.176, "read": 4.143}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.281, "count": 124, "max_ms": 9.845, "p95_ms": 8.022}, "gstreamer_push": {"avg_ms": 0.928, "count": 125, "max_ms": 2.135, "p95_ms": 1.025}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.051, "count": 124, "max_ms": 70.307, "p95_ms": 67.811}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.376, "count": 124, "max_ms": 91.144, "p95_ms": 88.544}, "yolo11_640_fire_11class": {"avg_ms": 53.519, "count": 124, "max_ms": 74.107, "p95_ms": 70.173}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.94, "count": 124, "max_ms": 94.652, "p95_ms": 90.156}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.073, "count": 125, "max_ms": 262.033, "p95_ms": 230.313}, "read_interval": {"avg_ms": 205.096, "count": 124, "max_ms": 272.335, "p95_ms": 238.558}, "triton_wait": {"avg_ms": 67.404, "count": 124, "max_ms": 94.662, "p95_ms": 90.422}, "visualize": {"avg_ms": 8.921, "count": 124, "max_ms": 11.269, "p95_ms": 10.047}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 176}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4175.07, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56137bb8cf80] error while decoding MB 51 3, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:06:57+09:00", "last_error_summary": "[h264 @ 0x56137bb8cf80] error while decoding MB 51 3, bytestream -14", "last_exit_at": "2026-07-01T16:04:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.669, "last_started_at": "2026-07-01T16:04:52+09:00", "pid": 7298, "process": {"exists": false, "pid": 7298}, "restart_count": 37, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.207, "sample_count": 1450, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21122, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:06:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T16:06:57+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T16:06:57+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T16:06:57.245355+09:00", "mtime_epoch": 1782889617.2453554, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207260}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:06:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 873.66, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 154, "pushed_frames": 154, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.155, "pushed": 5.155, "read": 5.155}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.102, "count": 154, "max_ms": 10.933, "p95_ms": 9.066}, "gstreamer_push": {"avg_ms": 0.943, "count": 154, "max_ms": 2.029, "p95_ms": 1.256}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.629, "count": 154, "max_ms": 72.014, "p95_ms": 69.693}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.556, "count": 154, "max_ms": 98.31, "p95_ms": 90.59}, "yolo11_640_fire_11class": {"avg_ms": 50.706, "count": 154, "max_ms": 76.744, "p95_ms": 71.532}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.809, "count": 154, "max_ms": 97.585, "p95_ms": 93.752}}, "truncated_count": 0}, "processing_total": {"avg_ms": 185.184, "count": 154, "max_ms": 241.173, "p95_ms": 223.683}, "read_interval": {"avg_ms": 194.866, "count": 154, "max_ms": 251.586, "p95_ms": 234.113}, "triton_wait": {"avg_ms": 66.125, "count": 154, "max_ms": 98.325, "p95_ms": 93.765}, "visualize": {"avg_ms": 8.392, "count": 154, "max_ms": 12.718, "p95_ms": 10.213}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 176}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4174.387, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:06:57+09:00", "last_error_summary": "[h264 @ 0x56137bb8cf80] error while decoding MB 51 3, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:06:57+09:00", "last_error_summary": "[h264 @ 0x56137bb8cf80] error while decoding MB 51 3, bytestream -14", "last_exit_at": "2026-07-01T16:06:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.765, "last_started_at": "2026-07-01T16:04:52+09:00", "pid": 7298, "process": {"exists": false, "pid": 7298}, "restart_count": 37, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.207, "sample_count": 1451, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21123, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:06:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T16:06:57+09:00", "last_frame_read_age_sec": 0.149, "last_frame_read_at": "2026-07-01T16:06:57+09:00", "mjpeg": {"age_sec": 0.141, "exists": true, "mtime": "2026-07-01T16:06:57.245355+09:00", "mtime_epoch": 1782889617.2453554, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207260}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:06:57+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 873.66, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.172, "pushed": 5.172, "read": 5.138}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.106, "count": 154, "max_ms": 10.933, "p95_ms": 9.142}, "gstreamer_push": {"avg_ms": 0.943, "count": 155, "max_ms": 2.029, "p95_ms": 1.256}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.655, "count": 154, "max_ms": 72.014, "p95_ms": 69.693}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.5, "count": 154, "max_ms": 98.31, "p95_ms": 90.59}, "yolo11_640_fire_11class": {"avg_ms": 50.657, "count": 154, "max_ms": 76.744, "p95_ms": 71.532}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.755, "count": 154, "max_ms": 97.585, "p95_ms": 93.752}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.916, "count": 155, "max_ms": 241.173, "p95_ms": 223.683}, "read_interval": {"avg_ms": 194.866, "count": 154, "max_ms": 251.586, "p95_ms": 234.113}, "triton_wait": {"avg_ms": 66.071, "count": 154, "max_ms": 98.325, "p95_ms": 93.765}, "visualize": {"avg_ms": 8.376, "count": 154, "max_ms": 12.718, "p95_ms": 10.213}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 176}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4174.379, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:06:57+09:00", "last_error_summary": "[h264 @ 0x56137bb8cf80] error while decoding MB 51 3, bytestream -14", "last_exit_at": "2026-07-01T16:06:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.769, "last_started_at": "2026-07-01T16:04:52+09:00", "pid": 7298, "process": {"exists": false, "pid": 7298}, "restart_count": 37, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.207, "sample_count": 1451, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21123, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:06:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T16:06:57+09:00", "last_frame_read_age_sec": 0.153, "last_frame_read_at": "2026-07-01T16:06:57+09:00", "mjpeg": {"age_sec": 0.146, "exists": true, "mtime": "2026-07-01T16:06:57.245355+09:00", "mtime_epoch": 1782889617.2453554, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207260}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:06:57+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 873.66, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.171, "pushed": 5.171, "read": 5.137}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.106, "count": 154, "max_ms": 10.933, "p95_ms": 9.142}, "gstreamer_push": {"avg_ms": 0.943, "count": 155, "max_ms": 2.029, "p95_ms": 1.256}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.655, "count": 154, "max_ms": 72.014, "p95_ms": 69.693}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.5, "count": 154, "max_ms": 98.31, "p95_ms": 90.59}, "yolo11_640_fire_11class": {"avg_ms": 50.657, "count": 154, "max_ms": 76.744, "p95_ms": 71.532}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.755, "count": 154, "max_ms": 97.585, "p95_ms": 93.752}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.916, "count": 155, "max_ms": 241.173, "p95_ms": 223.683}, "read_interval": {"avg_ms": 194.866, "count": 154, "max_ms": 251.586, "p95_ms": 234.113}, "triton_wait": {"avg_ms": 66.071, "count": 154, "max_ms": 98.325, "p95_ms": 93.765}, "visualize": {"avg_ms": 8.376, "count": 154, "max_ms": 12.718, "p95_ms": 10.213}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 176}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4282.203, "limit_mb": 8796093022207.996}, "detail": {"pid": 7405, "restart_count": 38, "start_count": 39}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:06:57+09:00", "last_error_summary": "[h264 @ 0x56137bb8cf80] error while decoding MB 51 3, bytestream -14", "last_exit_at": "2026-07-01T16:06:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:06:59+09:00", "pid": 7405, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.164, "name": "ffmpeg", "pid": 7405, "rss_mb": 139.164, "state": "S (sleeping)", "threads": 106}, "restart_count": 38, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.217, "sample_count": 1427, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21123, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:06:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.302, "last_frame_pushed_at": "2026-07-01T16:06:57+09:00", "last_frame_read_age_sec": 4.447, "last_frame_read_at": "2026-07-01T16:06:57+09:00", "mjpeg": {"age_sec": 4.287, "exists": true, "mtime": "2026-07-01T16:06:57.397358+09:00", "mtime_epoch": 1782889617.397358, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196218}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:07:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 875.852, "name": "celery", "pid": 237, "rss_mb": 873.66, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.45, "pushed": 4.45, "read": 4.417}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.06, "count": 133, "max_ms": 10.933, "p95_ms": 9.142}, "gstreamer_push": {"avg_ms": 0.936, "count": 133, "max_ms": 2.029, "p95_ms": 1.129}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.037, "count": 133, "max_ms": 72.014, "p95_ms": 70.469}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.55, "count": 133, "max_ms": 98.31, "p95_ms": 90.59}, "yolo11_640_fire_11class": {"avg_ms": 50.57, "count": 133, "max_ms": 76.744, "p95_ms": 71.532}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.633, "count": 133, "max_ms": 96.448, "p95_ms": 93.752}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.058, "count": 133, "max_ms": 241.173, "p95_ms": 217.862}, "read_interval": {"avg_ms": 194.133, "count": 132, "max_ms": 251.586, "p95_ms": 226.298}, "triton_wait": {"avg_ms": 65.991, "count": 133, "max_ms": 98.325, "p95_ms": 93.765}, "visualize": {"avg_ms": 8.373, "count": 133, "max_ms": 12.718, "p95_ms": 10.311}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 181}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4174.918, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5568ddcfec80] error while decoding MB 10 19, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:09:04+09:00", "last_error_summary": "[h264 @ 0x5568ddcfec80] error while decoding MB 10 19, bytestream -7", "last_exit_at": "2026-07-01T16:06:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.744, "last_started_at": "2026-07-01T16:06:59+09:00", "pid": 7405, "process": {"exists": false, "pid": 7405}, "restart_count": 38, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.502, "sample_count": 1457, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21745, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:09:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T16:09:04+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T16:09:04+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T16:09:04.387463+09:00", "mtime_epoch": 1782889744.387463, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204670}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:09:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 873.715, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.174, "pushed": 5.174, "read": 5.174}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.864, "count": 154, "max_ms": 10.867, "p95_ms": 8.23}, "gstreamer_push": {"avg_ms": 0.979, "count": 155, "max_ms": 3.102, "p95_ms": 1.823}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.044, "count": 154, "max_ms": 69.207, "p95_ms": 61.813}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.836, "count": 154, "max_ms": 95.722, "p95_ms": 90.448}, "yolo11_640_fire_11class": {"avg_ms": 52.955, "count": 154, "max_ms": 73.727, "p95_ms": 69.916}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.324, "count": 154, "max_ms": 98.641, "p95_ms": 89.069}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.54, "count": 155, "max_ms": 229.761, "p95_ms": 218.74}, "read_interval": {"avg_ms": 194.132, "count": 155, "max_ms": 239.197, "p95_ms": 228.556}, "triton_wait": {"avg_ms": 67.032, "count": 154, "max_ms": 98.654, "p95_ms": 91.185}, "visualize": {"avg_ms": 8.542, "count": 154, "max_ms": 11.849, "p95_ms": 10.226}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 181}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4174.074, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:09:04+09:00", "last_error_summary": "[h264 @ 0x5568ddcfec80] error while decoding MB 10 19, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:09:04+09:00", "last_error_summary": "[h264 @ 0x5568ddcfec80] error while decoding MB 10 19, bytestream -7", "last_exit_at": "2026-07-01T16:09:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.918, "last_started_at": "2026-07-01T16:06:59+09:00", "pid": 7405, "process": {"exists": false, "pid": 7405}, "restart_count": 38, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.502, "sample_count": 1457, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:09:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:09:04+09:00", "last_frame_read_age_sec": 0.225, "last_frame_read_at": "2026-07-01T16:09:04+09:00", "mjpeg": {"age_sec": 0.214, "exists": true, "mtime": "2026-07-01T16:09:04.387463+09:00", "mtime_epoch": 1782889744.387463, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204670}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:09:04+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 873.715, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.176, "pushed": 5.176, "read": 5.143}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.875, "count": 154, "max_ms": 10.867, "p95_ms": 8.267}, "gstreamer_push": {"avg_ms": 0.979, "count": 155, "max_ms": 3.102, "p95_ms": 1.823}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.143, "count": 154, "max_ms": 69.207, "p95_ms": 61.813}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.979, "count": 154, "max_ms": 95.722, "p95_ms": 90.448}, "yolo11_640_fire_11class": {"avg_ms": 53.008, "count": 154, "max_ms": 73.727, "p95_ms": 69.916}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.388, "count": 154, "max_ms": 98.641, "p95_ms": 89.069}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.784, "count": 155, "max_ms": 229.761, "p95_ms": 220.225}, "read_interval": {"avg_ms": 194.153, "count": 154, "max_ms": 239.197, "p95_ms": 228.556}, "triton_wait": {"avg_ms": 67.156, "count": 154, "max_ms": 98.654, "p95_ms": 91.185}, "visualize": {"avg_ms": 8.552, "count": 154, "max_ms": 11.849, "p95_ms": 10.226}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 181}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4174.051, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:09:04+09:00", "last_error_summary": "[h264 @ 0x5568ddcfec80] error while decoding MB 10 19, bytestream -7", "last_exit_at": "2026-07-01T16:09:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.921, "last_started_at": "2026-07-01T16:06:59+09:00", "pid": 7405, "process": {"exists": false, "pid": 7405}, "restart_count": 38, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.502, "sample_count": 1457, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:09:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T16:09:04+09:00", "last_frame_read_age_sec": 0.228, "last_frame_read_at": "2026-07-01T16:09:04+09:00", "mjpeg": {"age_sec": 0.217, "exists": true, "mtime": "2026-07-01T16:09:04.387463+09:00", "mtime_epoch": 1782889744.387463, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204670}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:09:04+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 873.715, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.176, "pushed": 5.176, "read": 5.142}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.875, "count": 154, "max_ms": 10.867, "p95_ms": 8.267}, "gstreamer_push": {"avg_ms": 0.979, "count": 155, "max_ms": 3.102, "p95_ms": 1.823}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.143, "count": 154, "max_ms": 69.207, "p95_ms": 61.813}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.979, "count": 154, "max_ms": 95.722, "p95_ms": 90.448}, "yolo11_640_fire_11class": {"avg_ms": 53.008, "count": 154, "max_ms": 73.727, "p95_ms": 69.916}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.388, "count": 154, "max_ms": 98.641, "p95_ms": 89.069}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.784, "count": 155, "max_ms": 229.761, "p95_ms": 220.225}, "read_interval": {"avg_ms": 194.153, "count": 154, "max_ms": 239.197, "p95_ms": 228.556}, "triton_wait": {"avg_ms": 67.156, "count": 154, "max_ms": 98.654, "p95_ms": 91.185}, "visualize": {"avg_ms": 8.552, "count": 154, "max_ms": 11.849, "p95_ms": 10.226}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 181}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4290.672, "limit_mb": 8796093022207.996}, "detail": {"pid": 7512, "restart_count": 39, "start_count": 40}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:09:04+09:00", "last_error_summary": "[h264 @ 0x5568ddcfec80] error while decoding MB 10 19, bytestream -7", "last_exit_at": "2026-07-01T16:09:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:09:06+09:00", "pid": 7512, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.469, "name": "ffmpeg", "pid": 7512, "rss_mb": 147.469, "state": "S (sleeping)", "threads": 106}, "restart_count": 39, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.751, "sample_count": 1438, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 21746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:09:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.044, "last_frame_pushed_at": "2026-07-01T16:09:04+09:00", "last_frame_read_age_sec": 4.266, "last_frame_read_at": "2026-07-01T16:09:04+09:00", "mjpeg": {"age_sec": 4.027, "exists": true, "mtime": "2026-07-01T16:09:04.615467+09:00", "mtime_epoch": 1782889744.6154668, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203420}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:09:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 873.715, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.468, "pushed": 4.468, "read": 4.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.81, "count": 133, "max_ms": 10.867, "p95_ms": 8.115}, "gstreamer_push": {"avg_ms": 0.994, "count": 134, "max_ms": 3.102, "p95_ms": 1.832}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.705, "count": 133, "max_ms": 69.207, "p95_ms": 62.984}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.72, "count": 133, "max_ms": 95.722, "p95_ms": 90.511}, "yolo11_640_fire_11class": {"avg_ms": 53.632, "count": 133, "max_ms": 73.727, "p95_ms": 69.929}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.155, "count": 133, "max_ms": 98.641, "p95_ms": 89.618}}, "truncated_count": 0}, "processing_total": {"avg_ms": 185.205, "count": 134, "max_ms": 229.761, "p95_ms": 220.949}, "read_interval": {"avg_ms": 194.703, "count": 133, "max_ms": 239.197, "p95_ms": 230.793}, "triton_wait": {"avg_ms": 68.01, "count": 133, "max_ms": 98.654, "p95_ms": 91.245}, "visualize": {"avg_ms": 8.48, "count": 133, "max_ms": 11.849, "p95_ms": 10.242}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 186}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4171.188, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563f3b3dfc80] error while decoding MB 12 42, bytestream -18", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:11:06+09:00", "last_error_summary": "[h264 @ 0x563f3b3dfc80] error while decoding MB 12 42, bytestream -18", "last_exit_at": "2026-07-01T16:09:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.699, "last_started_at": "2026-07-01T16:09:06+09:00", "pid": 7512, "process": {"exists": false, "pid": 7512}, "restart_count": 39, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.337, "sample_count": 1476, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22344, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:11:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T16:11:06+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T16:11:06+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T16:11:06.297497+09:00", "mtime_epoch": 1782889866.2974968, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202393}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:11:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 868.281, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 156, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.202, "pushed": 5.202, "read": 5.202}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.604, "count": 155, "max_ms": 12.21, "p95_ms": 7.82}, "gstreamer_push": {"avg_ms": 0.966, "count": 156, "max_ms": 3.526, "p95_ms": 1.749}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.752, "count": 155, "max_ms": 79.004, "p95_ms": 59.268}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.909, "count": 155, "max_ms": 98.33, "p95_ms": 96.932}, "yolo11_640_fire_11class": {"avg_ms": 54.082, "count": 155, "max_ms": 77.372, "p95_ms": 73.947}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.363, "count": 155, "max_ms": 103.605, "p95_ms": 97.099}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.523, "count": 156, "max_ms": 255.944, "p95_ms": 218.559}, "read_interval": {"avg_ms": 193.084, "count": 156, "max_ms": 265.357, "p95_ms": 227.619}, "triton_wait": {"avg_ms": 70.131, "count": 155, "max_ms": 103.618, "p95_ms": 97.615}, "visualize": {"avg_ms": 8.113, "count": 155, "max_ms": 12.709, "p95_ms": 9.674}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 186}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4170.398, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:11:06+09:00", "last_error_summary": "[h264 @ 0x563f3b3dfc80] error while decoding MB 12 42, bytestream -18", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:11:06+09:00", "last_error_summary": "[h264 @ 0x563f3b3dfc80] error while decoding MB 12 42, bytestream -18", "last_exit_at": "2026-07-01T16:11:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.868, "last_started_at": "2026-07-01T16:09:06+09:00", "pid": 7512, "process": {"exists": false, "pid": 7512}, "restart_count": 39, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.337, "sample_count": 1477, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22345, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:11:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:11:06+09:00", "last_frame_read_age_sec": 0.219, "last_frame_read_at": "2026-07-01T16:11:06+09:00", "mjpeg": {"age_sec": 0.212, "exists": true, "mtime": "2026-07-01T16:11:06.297497+09:00", "mtime_epoch": 1782889866.2974968, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202393}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:11:06+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 868.281, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.205, "pushed": 5.205, "read": 5.172}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.602, "count": 155, "max_ms": 12.21, "p95_ms": 7.82}, "gstreamer_push": {"avg_ms": 0.966, "count": 156, "max_ms": 3.526, "p95_ms": 1.749}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.801, "count": 155, "max_ms": 79.004, "p95_ms": 59.268}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.179, "count": 155, "max_ms": 98.33, "p95_ms": 96.932}, "yolo11_640_fire_11class": {"avg_ms": 54.166, "count": 155, "max_ms": 77.372, "p95_ms": 73.947}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.591, "count": 155, "max_ms": 103.605, "p95_ms": 97.099}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.731, "count": 156, "max_ms": 255.944, "p95_ms": 218.559}, "read_interval": {"avg_ms": 193.1, "count": 155, "max_ms": 265.357, "p95_ms": 227.619}, "triton_wait": {"avg_ms": 70.37, "count": 155, "max_ms": 103.618, "p95_ms": 97.615}, "visualize": {"avg_ms": 8.115, "count": 155, "max_ms": 12.709, "p95_ms": 9.674}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 186}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4170.391, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:11:06+09:00", "last_error_summary": "[h264 @ 0x563f3b3dfc80] error while decoding MB 12 42, bytestream -18", "last_exit_at": "2026-07-01T16:11:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.871, "last_started_at": "2026-07-01T16:09:06+09:00", "pid": 7512, "process": {"exists": false, "pid": 7512}, "restart_count": 39, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.337, "sample_count": 1477, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22345, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:11:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:11:06+09:00", "last_frame_read_age_sec": 0.223, "last_frame_read_at": "2026-07-01T16:11:06+09:00", "mjpeg": {"age_sec": 0.215, "exists": true, "mtime": "2026-07-01T16:11:06.297497+09:00", "mtime_epoch": 1782889866.2974968, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202393}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:11:06+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 868.281, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.205, "pushed": 5.205, "read": 5.172}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.602, "count": 155, "max_ms": 12.21, "p95_ms": 7.82}, "gstreamer_push": {"avg_ms": 0.966, "count": 156, "max_ms": 3.526, "p95_ms": 1.749}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.801, "count": 155, "max_ms": 79.004, "p95_ms": 59.268}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.179, "count": 155, "max_ms": 98.33, "p95_ms": 96.932}, "yolo11_640_fire_11class": {"avg_ms": 54.166, "count": 155, "max_ms": 77.372, "p95_ms": 73.947}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.591, "count": 155, "max_ms": 103.605, "p95_ms": 97.099}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.731, "count": 156, "max_ms": 255.944, "p95_ms": 218.559}, "read_interval": {"avg_ms": 193.1, "count": 155, "max_ms": 265.357, "p95_ms": 227.619}, "triton_wait": {"avg_ms": 70.37, "count": 155, "max_ms": 103.618, "p95_ms": 97.615}, "visualize": {"avg_ms": 8.115, "count": 155, "max_ms": 12.709, "p95_ms": 9.674}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 186}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4286.488, "limit_mb": 8796093022207.996}, "detail": {"pid": 7619, "restart_count": 40, "start_count": 41}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:11:06+09:00", "last_error_summary": "[h264 @ 0x563f3b3dfc80] error while decoding MB 12 42, bytestream -18", "last_exit_at": "2026-07-01T16:11:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:11:08+09:00", "pid": 7619, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.074, "name": "ffmpeg", "pid": 7619, "rss_mb": 147.074, "state": "S (sleeping)", "threads": 106}, "restart_count": 40, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 224.808, "sample_count": 1455, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22345, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:11:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.108, "last_frame_pushed_at": "2026-07-01T16:11:06+09:00", "last_frame_read_age_sec": 4.325, "last_frame_read_at": "2026-07-01T16:11:06+09:00", "mjpeg": {"age_sec": 4.093, "exists": true, "mtime": "2026-07-01T16:11:06.521501+09:00", "mtime_epoch": 1782889866.5215006, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202439}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:11:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 868.281, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.578, "pushed": 4.578, "read": 4.544}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.586, "count": 136, "max_ms": 12.21, "p95_ms": 7.897}, "gstreamer_push": {"avg_ms": 0.979, "count": 137, "max_ms": 3.526, "p95_ms": 1.761}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.851, "count": 136, "max_ms": 76.794, "p95_ms": 58.599}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.507, "count": 136, "max_ms": 98.249, "p95_ms": 95.567}, "yolo11_640_fire_11class": {"avg_ms": 52.505, "count": 136, "max_ms": 75.771, "p95_ms": 73.088}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.484, "count": 136, "max_ms": 103.605, "p95_ms": 95.416}}, "truncated_count": 0}, "processing_total": {"avg_ms": 180.343, "count": 137, "max_ms": 255.944, "p95_ms": 216.386}, "read_interval": {"avg_ms": 189.754, "count": 136, "max_ms": 265.357, "p95_ms": 226.057}, "triton_wait": {"avg_ms": 67.89, "count": 136, "max_ms": 103.618, "p95_ms": 97.028}, "visualize": {"avg_ms": 8.069, "count": 136, "max_ms": 12.709, "p95_ms": 9.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4175.305, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a059892900] error while decoding MB 37 23, bytestream -8", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:13:10+09:00", "last_error_summary": "[h264 @ 0x55a059892900] error while decoding MB 37 23, bytestream -8", "last_exit_at": "2026-07-01T16:11:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.605, "last_started_at": "2026-07-01T16:11:08+09:00", "pid": 7619, "process": {"exists": false, "pid": 7619}, "restart_count": 40, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.023, "sample_count": 1483, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22953, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:13:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T16:13:10+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T16:13:10+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T16:13:10.171573+09:00", "mtime_epoch": 1782889990.1715734, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206532}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:13:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 871.535, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.177, "pushed": 5.177, "read": 5.177}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.62, "count": 154, "max_ms": 10.206, "p95_ms": 7.677}, "gstreamer_push": {"avg_ms": 1.002, "count": 155, "max_ms": 3.115, "p95_ms": 1.849}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.699, "count": 154, "max_ms": 71.818, "p95_ms": 65.666}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.612, "count": 154, "max_ms": 90.938, "p95_ms": 88.43}, "yolo11_640_fire_11class": {"avg_ms": 55.189, "count": 154, "max_ms": 75.786, "p95_ms": 72.079}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.619, "count": 154, "max_ms": 92.066, "p95_ms": 89.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.826, "count": 155, "max_ms": 252.865, "p95_ms": 228.141}, "read_interval": {"avg_ms": 193.876, "count": 155, "max_ms": 260.53, "p95_ms": 236.339}, "triton_wait": {"avg_ms": 69.024, "count": 154, "max_ms": 92.079, "p95_ms": 89.975}, "visualize": {"avg_ms": 8.208, "count": 154, "max_ms": 10.092, "p95_ms": 9.92}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4167.926, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:13:10+09:00", "last_error_summary": "[h264 @ 0x55a059892900] error while decoding MB 37 23, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:13:10+09:00", "last_error_summary": "[h264 @ 0x55a059892900] error while decoding MB 37 23, bytestream -8", "last_exit_at": "2026-07-01T16:13:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.775, "last_started_at": "2026-07-01T16:11:08+09:00", "pid": 7619, "process": {"exists": false, "pid": 7619}, "restart_count": 40, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.023, "sample_count": 1484, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22954, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:13:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T16:13:10+09:00", "last_frame_read_age_sec": 0.222, "last_frame_read_at": "2026-07-01T16:13:10+09:00", "mjpeg": {"age_sec": 0.217, "exists": true, "mtime": "2026-07-01T16:13:10.171573+09:00", "mtime_epoch": 1782889990.1715734, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206532}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:13:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 865.348, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.178, "pushed": 5.178, "read": 5.145}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.619, "count": 154, "max_ms": 10.206, "p95_ms": 7.677}, "gstreamer_push": {"avg_ms": 1.008, "count": 155, "max_ms": 3.115, "p95_ms": 1.852}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.88, "count": 154, "max_ms": 71.818, "p95_ms": 65.961}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.836, "count": 154, "max_ms": 90.938, "p95_ms": 88.704}, "yolo11_640_fire_11class": {"avg_ms": 55.345, "count": 154, "max_ms": 75.786, "p95_ms": 72.159}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.794, "count": 154, "max_ms": 92.066, "p95_ms": 89.939}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.182, "count": 155, "max_ms": 252.865, "p95_ms": 228.141}, "read_interval": {"avg_ms": 194.036, "count": 154, "max_ms": 260.53, "p95_ms": 236.339}, "triton_wait": {"avg_ms": 69.2, "count": 154, "max_ms": 92.079, "p95_ms": 89.979}, "visualize": {"avg_ms": 8.216, "count": 154, "max_ms": 10.092, "p95_ms": 9.92}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4167.707, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:13:10+09:00", "last_error_summary": "[h264 @ 0x55a059892900] error while decoding MB 37 23, bytestream -8", "last_exit_at": "2026-07-01T16:13:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.779, "last_started_at": "2026-07-01T16:11:08+09:00", "pid": 7619, "process": {"exists": false, "pid": 7619}, "restart_count": 40, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.023, "sample_count": 1484, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22954, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:13:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T16:13:10+09:00", "last_frame_read_age_sec": 0.225, "last_frame_read_at": "2026-07-01T16:13:10+09:00", "mjpeg": {"age_sec": 0.221, "exists": true, "mtime": "2026-07-01T16:13:10.171573+09:00", "mtime_epoch": 1782889990.1715734, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206532}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:13:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 865.348, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 154, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.178, "pushed": 5.178, "read": 5.144}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.619, "count": 154, "max_ms": 10.206, "p95_ms": 7.677}, "gstreamer_push": {"avg_ms": 1.008, "count": 155, "max_ms": 3.115, "p95_ms": 1.852}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.88, "count": 154, "max_ms": 71.818, "p95_ms": 65.961}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.836, "count": 154, "max_ms": 90.938, "p95_ms": 88.704}, "yolo11_640_fire_11class": {"avg_ms": 55.345, "count": 154, "max_ms": 75.786, "p95_ms": 72.159}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.794, "count": 154, "max_ms": 92.066, "p95_ms": 89.939}}, "truncated_count": 0}, "processing_total": {"avg_ms": 184.182, "count": 155, "max_ms": 252.865, "p95_ms": 228.141}, "read_interval": {"avg_ms": 194.036, "count": 154, "max_ms": 260.53, "p95_ms": 236.339}, "triton_wait": {"avg_ms": 69.2, "count": 154, "max_ms": 92.079, "p95_ms": 89.979}, "visualize": {"avg_ms": 8.216, "count": 154, "max_ms": 10.092, "p95_ms": 9.92}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4273.785, "limit_mb": 8796093022207.996}, "detail": {"pid": 7726, "restart_count": 41, "start_count": 42}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:13:10+09:00", "last_error_summary": "[h264 @ 0x55a059892900] error while decoding MB 37 23, bytestream -8", "last_exit_at": "2026-07-01T16:13:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:13:13+09:00", "pid": 7726, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.527, "name": "ffmpeg", "pid": 7726, "rss_mb": 136.527, "state": "S (sleeping)", "threads": 106}, "restart_count": 41, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 226.023, "sample_count": 1461, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 22954, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:13:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.83, "last_frame_pushed_at": "2026-07-01T16:13:10+09:00", "last_frame_read_age_sec": 5.047, "last_frame_read_at": "2026-07-01T16:13:10+09:00", "mjpeg": {"age_sec": 4.815, "exists": true, "mtime": "2026-07-01T16:13:10.399577+09:00", "mtime_epoch": 1782889990.3995771, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206794}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:13:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 865.348, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.391, "pushed": 4.391, "read": 4.358}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.556, "count": 131, "max_ms": 7.783, "p95_ms": 7.671}, "gstreamer_push": {"avg_ms": 1.012, "count": 131, "max_ms": 3.115, "p95_ms": 1.852}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.674, "count": 131, "max_ms": 71.818, "p95_ms": 65.269}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.729, "count": 131, "max_ms": 90.938, "p95_ms": 88.37}, "yolo11_640_fire_11class": {"avg_ms": 55.279, "count": 131, "max_ms": 75.786, "p95_ms": 72.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.665, "count": 131, "max_ms": 92.066, "p95_ms": 89.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 182.322, "count": 131, "max_ms": 248.075, "p95_ms": 222.421}, "read_interval": {"avg_ms": 192.19, "count": 130, "max_ms": 258.81, "p95_ms": 231.838}, "triton_wait": {"avg_ms": 69.082, "count": 131, "max_ms": 92.079, "p95_ms": 89.975}, "visualize": {"avg_ms": 8.128, "count": 131, "max_ms": 10.092, "p95_ms": 9.617}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.831, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 196}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4173.062, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562093ff5980] error while decoding MB 22 36, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:15:21+09:00", "last_error_summary": "[h264 @ 0x562093ff5980] error while decoding MB 22 36, bytestream -7", "last_exit_at": "2026-07-01T16:13:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.725, "last_started_at": "2026-07-01T16:13:13+09:00", "pid": 7726, "process": {"exists": false, "pid": 7726}, "restart_count": 41, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.776, "sample_count": 1455, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23566, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:15:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T16:15:21+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-01T16:15:21+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-01T16:15:21.881790+09:00", "mtime_epoch": 1782890121.88179, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206551}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:15:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 869.191, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.546, "pushed": 4.546, "read": 4.546}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.386, "count": 135, "max_ms": 10.585, "p95_ms": 9.87}, "gstreamer_push": {"avg_ms": 0.935, "count": 136, "max_ms": 1.93, "p95_ms": 1.806}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.812, "count": 135, "max_ms": 75.792, "p95_ms": 56.046}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.449, "count": 135, "max_ms": 87.707, "p95_ms": 85.415}, "yolo11_640_fire_11class": {"avg_ms": 50.784, "count": 135, "max_ms": 75.241, "p95_ms": 70.708}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.825, "count": 135, "max_ms": 95.123, "p95_ms": 90.733}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.607, "count": 136, "max_ms": 291.997, "p95_ms": 241.468}, "read_interval": {"avg_ms": 221.315, "count": 136, "max_ms": 302.308, "p95_ms": 251.103}, "triton_wait": {"avg_ms": 64.915, "count": 135, "max_ms": 95.136, "p95_ms": 90.745}, "visualize": {"avg_ms": 9.981, "count": 135, "max_ms": 11.959, "p95_ms": 11.194}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 196}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4172.66, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:15:21+09:00", "last_error_summary": "[h264 @ 0x562093ff5980] error while decoding MB 22 36, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:15:21+09:00", "last_error_summary": "[h264 @ 0x562093ff5980] error while decoding MB 22 36, bytestream -7", "last_exit_at": "2026-07-01T16:15:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.9, "last_started_at": "2026-07-01T16:13:13+09:00", "pid": 7726, "process": {"exists": false, "pid": 7726}, "restart_count": 41, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.776, "sample_count": 1455, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23567, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:15:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:15:22+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T16:15:21+09:00", "mjpeg": {"age_sec": 0.232, "exists": true, "mtime": "2026-07-01T16:15:21.881790+09:00", "mtime_epoch": 1782890121.88179, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206551}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:15:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 869.191, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.552, "pushed": 4.552, "read": 4.519}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.384, "count": 135, "max_ms": 10.585, "p95_ms": 9.87}, "gstreamer_push": {"avg_ms": 0.934, "count": 136, "max_ms": 1.93, "p95_ms": 1.806}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.822, "count": 135, "max_ms": 75.792, "p95_ms": 56.046}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.476, "count": 135, "max_ms": 87.707, "p95_ms": 85.415}, "yolo11_640_fire_11class": {"avg_ms": 50.793, "count": 135, "max_ms": 75.241, "p95_ms": 70.708}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.838, "count": 135, "max_ms": 95.123, "p95_ms": 90.733}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.509, "count": 136, "max_ms": 291.997, "p95_ms": 241.468}, "read_interval": {"avg_ms": 221.103, "count": 135, "max_ms": 302.308, "p95_ms": 251.103}, "triton_wait": {"avg_ms": 64.928, "count": 135, "max_ms": 95.136, "p95_ms": 90.745}, "visualize": {"avg_ms": 9.997, "count": 135, "max_ms": 13.05, "p95_ms": 11.239}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.875, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 196}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4172.652, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:15:21+09:00", "last_error_summary": "[h264 @ 0x562093ff5980] error while decoding MB 22 36, bytestream -7", "last_exit_at": "2026-07-01T16:15:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.904, "last_started_at": "2026-07-01T16:13:13+09:00", "pid": 7726, "process": {"exists": false, "pid": 7726}, "restart_count": 41, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.776, "sample_count": 1455, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23567, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:15:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T16:15:22+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T16:15:21+09:00", "mjpeg": {"age_sec": 0.235, "exists": true, "mtime": "2026-07-01T16:15:21.881790+09:00", "mtime_epoch": 1782890121.88179, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206551}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:15:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 869.191, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.552, "pushed": 4.552, "read": 4.518}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.384, "count": 135, "max_ms": 10.585, "p95_ms": 9.87}, "gstreamer_push": {"avg_ms": 0.934, "count": 136, "max_ms": 1.93, "p95_ms": 1.806}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.822, "count": 135, "max_ms": 75.792, "p95_ms": 56.046}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.476, "count": 135, "max_ms": 87.707, "p95_ms": 85.415}, "yolo11_640_fire_11class": {"avg_ms": 50.793, "count": 135, "max_ms": 75.241, "p95_ms": 70.708}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.838, "count": 135, "max_ms": 95.123, "p95_ms": 90.733}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.509, "count": 136, "max_ms": 291.997, "p95_ms": 241.468}, "read_interval": {"avg_ms": 221.103, "count": 135, "max_ms": 302.308, "p95_ms": 251.103}, "triton_wait": {"avg_ms": 64.928, "count": 135, "max_ms": 95.136, "p95_ms": 90.745}, "visualize": {"avg_ms": 9.997, "count": 135, "max_ms": 13.05, "p95_ms": 11.239}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 196}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4281.641, "limit_mb": 8796093022207.996}, "detail": {"pid": 7833, "restart_count": 42, "start_count": 43}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:15:21+09:00", "last_error_summary": "[h264 @ 0x562093ff5980] error while decoding MB 22 36, bytestream -7", "last_exit_at": "2026-07-01T16:15:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:15:24+09:00", "pid": 7833, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.602, "name": "ffmpeg", "pid": 7833, "rss_mb": 141.602, "state": "S (sleeping)", "threads": 106}, "restart_count": 42, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.789, "sample_count": 1431, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 23567, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:15:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.584, "last_frame_pushed_at": "2026-07-01T16:15:22+09:00", "last_frame_read_age_sec": 4.812, "last_frame_read_at": "2026-07-01T16:15:21+09:00", "mjpeg": {"age_sec": 4.565, "exists": true, "mtime": "2026-07-01T16:15:22.129794+09:00", "mtime_epoch": 1782890122.1297941, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206983}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:15:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 869.191, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.817, "pushed": 3.817, "read": 3.783}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.478, "count": 114, "max_ms": 10.585, "p95_ms": 9.922}, "gstreamer_push": {"avg_ms": 0.927, "count": 114, "max_ms": 1.93, "p95_ms": 1.806}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.028, "count": 114, "max_ms": 75.792, "p95_ms": 56.263}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.705, "count": 114, "max_ms": 87.033, "p95_ms": 85.415}, "yolo11_640_fire_11class": {"avg_ms": 50.861, "count": 114, "max_ms": 75.241, "p95_ms": 70.708}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.182, "count": 114, "max_ms": 95.123, "p95_ms": 92.028}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.619, "count": 114, "max_ms": 291.997, "p95_ms": 241.468}, "read_interval": {"avg_ms": 223.326, "count": 113, "max_ms": 302.308, "p95_ms": 251.103}, "triton_wait": {"avg_ms": 65.211, "count": 114, "max_ms": 95.136, "p95_ms": 92.041}, "visualize": {"avg_ms": 10.096, "count": 114, "max_ms": 13.05, "p95_ms": 11.303}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 201}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4170.371, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5557a8d627c0] error while decoding MB 59 24, bytestream -24", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:17:28+09:00", "last_error_summary": "[h264 @ 0x5557a8d627c0] error while decoding MB 59 24, bytestream -24", "last_exit_at": "2026-07-01T16:15:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.276, "last_started_at": "2026-07-01T16:15:24+09:00", "pid": 7833, "process": {"exists": false, "pid": 7833}, "restart_count": 42, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.094, "sample_count": 1433, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24172, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:17:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T16:17:28+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T16:17:28+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T16:17:28.923935+09:00", "mtime_epoch": 1782890248.9239347, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:17:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 866.098, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.945, "pushed": 4.945, "read": 4.945}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.31, "count": 147, "max_ms": 9.335, "p95_ms": 8.561}, "gstreamer_push": {"avg_ms": 0.961, "count": 148, "max_ms": 3.008, "p95_ms": 1.589}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.332, "count": 147, "max_ms": 70.389, "p95_ms": 59.604}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.31, "count": 147, "max_ms": 97.965, "p95_ms": 91.47}, "yolo11_640_fire_11class": {"avg_ms": 52.228, "count": 147, "max_ms": 80.396, "p95_ms": 74.541}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.843, "count": 147, "max_ms": 101.333, "p95_ms": 93.408}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.508, "count": 148, "max_ms": 284.658, "p95_ms": 226.665}, "read_interval": {"avg_ms": 203.134, "count": 148, "max_ms": 295.067, "p95_ms": 237.145}, "triton_wait": {"avg_ms": 66.244, "count": 147, "max_ms": 101.346, "p95_ms": 93.42}, "visualize": {"avg_ms": 8.951, "count": 147, "max_ms": 11.058, "p95_ms": 10.108}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 201}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4170.227, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:17:28+09:00", "last_error_summary": "[h264 @ 0x5557a8d627c0] error while decoding MB 59 24, bytestream -24", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:17:28+09:00", "last_error_summary": "[h264 @ 0x5557a8d627c0] error while decoding MB 59 24, bytestream -24", "last_exit_at": "2026-07-01T16:17:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.419, "last_started_at": "2026-07-01T16:15:24+09:00", "pid": 7833, "process": {"exists": false, "pid": 7833}, "restart_count": 42, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.094, "sample_count": 1433, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24173, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:17:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:17:29+09:00", "last_frame_read_age_sec": 0.197, "last_frame_read_at": "2026-07-01T16:17:28+09:00", "mjpeg": {"age_sec": 0.189, "exists": true, "mtime": "2026-07-01T16:17:28.923935+09:00", "mtime_epoch": 1782890248.9239347, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:17:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 866.871, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.954, "pushed": 4.954, "read": 4.92}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.307, "count": 148, "max_ms": 9.335, "p95_ms": 8.561}, "gstreamer_push": {"avg_ms": 0.961, "count": 148, "max_ms": 3.008, "p95_ms": 1.589}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.346, "count": 148, "max_ms": 70.389, "p95_ms": 59.604}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.325, "count": 148, "max_ms": 97.965, "p95_ms": 91.47}, "yolo11_640_fire_11class": {"avg_ms": 52.259, "count": 148, "max_ms": 80.396, "p95_ms": 74.541}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.861, "count": 148, "max_ms": 101.333, "p95_ms": 93.408}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.536, "count": 148, "max_ms": 284.658, "p95_ms": 226.665}, "read_interval": {"avg_ms": 203.162, "count": 147, "max_ms": 295.067, "p95_ms": 237.145}, "triton_wait": {"avg_ms": 66.259, "count": 148, "max_ms": 101.346, "p95_ms": 93.42}, "visualize": {"avg_ms": 8.95, "count": 148, "max_ms": 11.058, "p95_ms": 10.108}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 201}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4170.219, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:17:28+09:00", "last_error_summary": "[h264 @ 0x5557a8d627c0] error while decoding MB 59 24, bytestream -24", "last_exit_at": "2026-07-01T16:17:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.423, "last_started_at": "2026-07-01T16:15:24+09:00", "pid": 7833, "process": {"exists": false, "pid": 7833}, "restart_count": 42, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.094, "sample_count": 1433, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24173, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:17:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:17:29+09:00", "last_frame_read_age_sec": 0.2, "last_frame_read_at": "2026-07-01T16:17:28+09:00", "mjpeg": {"age_sec": 0.192, "exists": true, "mtime": "2026-07-01T16:17:28.923935+09:00", "mtime_epoch": 1782890248.9239347, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:17:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 866.871, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.953, "pushed": 4.953, "read": 4.92}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.307, "count": 148, "max_ms": 9.335, "p95_ms": 8.561}, "gstreamer_push": {"avg_ms": 0.961, "count": 148, "max_ms": 3.008, "p95_ms": 1.589}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.346, "count": 148, "max_ms": 70.389, "p95_ms": 59.604}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.325, "count": 148, "max_ms": 97.965, "p95_ms": 91.47}, "yolo11_640_fire_11class": {"avg_ms": 52.259, "count": 148, "max_ms": 80.396, "p95_ms": 74.541}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.861, "count": 148, "max_ms": 101.333, "p95_ms": 93.408}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.536, "count": 148, "max_ms": 284.658, "p95_ms": 226.665}, "read_interval": {"avg_ms": 203.162, "count": 147, "max_ms": 295.067, "p95_ms": 237.145}, "triton_wait": {"avg_ms": 66.259, "count": 148, "max_ms": 101.346, "p95_ms": 93.42}, "visualize": {"avg_ms": 8.95, "count": 148, "max_ms": 11.058, "p95_ms": 10.108}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 201}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4275.719, "limit_mb": 8796093022207.996}, "detail": {"pid": 7940, "restart_count": 43, "start_count": 44}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:17:28+09:00", "last_error_summary": "[h264 @ 0x5557a8d627c0] error while decoding MB 59 24, bytestream -24", "last_exit_at": "2026-07-01T16:17:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:17:31+09:00", "pid": 7940, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.457, "name": "ffmpeg", "pid": 7940, "rss_mb": 136.457, "state": "S (sleeping)", "threads": 106}, "restart_count": 43, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 231.776, "sample_count": 1409, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24173, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:17:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.559, "last_frame_pushed_at": "2026-07-01T16:17:29+09:00", "last_frame_read_age_sec": 4.753, "last_frame_read_at": "2026-07-01T16:17:28+09:00", "mjpeg": {"age_sec": 4.541, "exists": true, "mtime": "2026-07-01T16:17:29.127938+09:00", "mtime_epoch": 1782890249.127938, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202607}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:17:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 866.871, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.17, "pushed": 4.17, "read": 4.136}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.415, "count": 124, "max_ms": 9.335, "p95_ms": 8.572}, "gstreamer_push": {"avg_ms": 0.943, "count": 125, "max_ms": 2.97, "p95_ms": 1.322}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.413, "count": 124, "max_ms": 70.389, "p95_ms": 61.351}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.018, "count": 124, "max_ms": 97.965, "p95_ms": 92.191}, "yolo11_640_fire_11class": {"avg_ms": 52.418, "count": 124, "max_ms": 77.48, "p95_ms": 74.541}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.171, "count": 124, "max_ms": 99.653, "p95_ms": 93.408}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.672, "count": 125, "max_ms": 284.658, "p95_ms": 226.665}, "read_interval": {"avg_ms": 205.267, "count": 124, "max_ms": 295.067, "p95_ms": 237.145}, "triton_wait": {"avg_ms": 66.643, "count": 124, "max_ms": 99.67, "p95_ms": 93.42}, "visualize": {"avg_ms": 9.047, "count": 124, "max_ms": 11.058, "p95_ms": 10.149}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 206}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4177.719, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fd877a4480] error while decoding MB 50 5, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:19:35+09:00", "last_error_summary": "[h264 @ 0x55fd877a4480] error while decoding MB 50 5, bytestream -13", "last_exit_at": "2026-07-01T16:17:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.93, "last_started_at": "2026-07-01T16:17:31+09:00", "pid": 7940, "process": {"exists": false, "pid": 7940}, "restart_count": 43, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.605, "sample_count": 1386, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24737, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:19:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T16:19:35+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T16:19:35+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T16:19:35.558078+09:00", "mtime_epoch": 1782890375.5580776, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203980}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:19:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 872.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.887, "pushed": 4.887, "read": 4.887}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.463, "count": 146, "max_ms": 9.596, "p95_ms": 9.292}, "gstreamer_push": {"avg_ms": 0.946, "count": 146, "max_ms": 3.019, "p95_ms": 1.024}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.492, "count": 146, "max_ms": 77.22, "p95_ms": 65.719}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.026, "count": 146, "max_ms": 95.097, "p95_ms": 89.723}, "yolo11_640_fire_11class": {"avg_ms": 53.325, "count": 146, "max_ms": 85.494, "p95_ms": 73.933}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.939, "count": 146, "max_ms": 102.222, "p95_ms": 94.552}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.758, "count": 146, "max_ms": 292.322, "p95_ms": 229.16}, "read_interval": {"avg_ms": 205.384, "count": 146, "max_ms": 305.285, "p95_ms": 239.463}, "triton_wait": {"avg_ms": 68.032, "count": 146, "max_ms": 102.235, "p95_ms": 94.566}, "visualize": {"avg_ms": 9.088, "count": 146, "max_ms": 13.26, "p95_ms": 10.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 206}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4177.008, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:19:35+09:00", "last_error_summary": "[h264 @ 0x55fd877a4480] error while decoding MB 50 5, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:19:35+09:00", "last_error_summary": "[h264 @ 0x55fd877a4480] error while decoding MB 50 5, bytestream -13", "last_exit_at": "2026-07-01T16:19:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.071, "last_started_at": "2026-07-01T16:17:31+09:00", "pid": 7940, "process": {"exists": false, "pid": 7940}, "restart_count": 43, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.605, "sample_count": 1386, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24738, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:19:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T16:19:35+09:00", "last_frame_read_age_sec": 0.192, "last_frame_read_at": "2026-07-01T16:19:35+09:00", "mjpeg": {"age_sec": 0.181, "exists": true, "mtime": "2026-07-01T16:19:35.558078+09:00", "mtime_epoch": 1782890375.5580776, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203980}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:19:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 872.426, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.893, "pushed": 4.893, "read": 4.859}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.468, "count": 146, "max_ms": 9.596, "p95_ms": 9.292}, "gstreamer_push": {"avg_ms": 0.945, "count": 146, "max_ms": 3.019, "p95_ms": 1.024}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.415, "count": 146, "max_ms": 77.22, "p95_ms": 65.719}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.993, "count": 146, "max_ms": 95.097, "p95_ms": 89.723}, "yolo11_640_fire_11class": {"avg_ms": 53.303, "count": 146, "max_ms": 85.494, "p95_ms": 73.933}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.939, "count": 146, "max_ms": 102.222, "p95_ms": 94.552}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.925, "count": 146, "max_ms": 292.322, "p95_ms": 229.16}, "read_interval": {"avg_ms": 205.618, "count": 145, "max_ms": 305.285, "p95_ms": 239.463}, "triton_wait": {"avg_ms": 68.033, "count": 146, "max_ms": 102.235, "p95_ms": 94.566}, "visualize": {"avg_ms": 9.097, "count": 146, "max_ms": 13.26, "p95_ms": 10.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.841, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 206}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4177.0, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:19:35+09:00", "last_error_summary": "[h264 @ 0x55fd877a4480] error while decoding MB 50 5, bytestream -13", "last_exit_at": "2026-07-01T16:19:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.074, "last_started_at": "2026-07-01T16:17:31+09:00", "pid": 7940, "process": {"exists": false, "pid": 7940}, "restart_count": 43, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.605, "sample_count": 1386, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24738, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:19:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T16:19:35+09:00", "last_frame_read_age_sec": 0.195, "last_frame_read_at": "2026-07-01T16:19:35+09:00", "mjpeg": {"age_sec": 0.184, "exists": true, "mtime": "2026-07-01T16:19:35.558078+09:00", "mtime_epoch": 1782890375.5580776, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203980}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:19:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 872.426, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.892, "pushed": 4.892, "read": 4.858}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.468, "count": 146, "max_ms": 9.596, "p95_ms": 9.292}, "gstreamer_push": {"avg_ms": 0.945, "count": 146, "max_ms": 3.019, "p95_ms": 1.024}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.415, "count": 146, "max_ms": 77.22, "p95_ms": 65.719}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.993, "count": 146, "max_ms": 95.097, "p95_ms": 89.723}, "yolo11_640_fire_11class": {"avg_ms": 53.303, "count": 146, "max_ms": 85.494, "p95_ms": 73.933}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.939, "count": 146, "max_ms": 102.222, "p95_ms": 94.552}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.925, "count": 146, "max_ms": 292.322, "p95_ms": 229.16}, "read_interval": {"avg_ms": 205.618, "count": 145, "max_ms": 305.285, "p95_ms": 239.463}, "triton_wait": {"avg_ms": 68.033, "count": 146, "max_ms": 102.235, "p95_ms": 94.566}, "visualize": {"avg_ms": 9.097, "count": 146, "max_ms": 13.26, "p95_ms": 10.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 206}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4282.473, "limit_mb": 8796093022207.996}, "detail": {"pid": 8047, "restart_count": 44, "start_count": 45}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:19:35+09:00", "last_error_summary": "[h264 @ 0x55fd877a4480] error while decoding MB 50 5, bytestream -13", "last_exit_at": "2026-07-01T16:19:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:19:38+09:00", "pid": 8047, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.414, "name": "ffmpeg", "pid": 8047, "rss_mb": 136.414, "state": "S (sleeping)", "threads": 106}, "restart_count": 44, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.617, "sample_count": 1363, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24738, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:19:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.526, "last_frame_pushed_at": "2026-07-01T16:19:35+09:00", "last_frame_read_age_sec": 4.713, "last_frame_read_at": "2026-07-01T16:19:35+09:00", "mjpeg": {"age_sec": 4.51, "exists": true, "mtime": "2026-07-01T16:19:35.750081+09:00", "mtime_epoch": 1782890375.7500808, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:19:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 872.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.125, "pushed": 4.125, "read": 4.091}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.603, "count": 123, "max_ms": 9.596, "p95_ms": 9.311}, "gstreamer_push": {"avg_ms": 0.928, "count": 123, "max_ms": 1.936, "p95_ms": 1.006}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.186, "count": 123, "max_ms": 77.22, "p95_ms": 65.376}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.536, "count": 123, "max_ms": 95.097, "p95_ms": 88.987}, "yolo11_640_fire_11class": {"avg_ms": 53.184, "count": 123, "max_ms": 85.494, "p95_ms": 73.933}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.611, "count": 123, "max_ms": 102.222, "p95_ms": 94.552}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.671, "count": 123, "max_ms": 284.704, "p95_ms": 229.16}, "read_interval": {"avg_ms": 207.437, "count": 122, "max_ms": 295.27, "p95_ms": 239.463}, "triton_wait": {"avg_ms": 67.702, "count": 123, "max_ms": 102.235, "p95_ms": 94.566}, "visualize": {"avg_ms": 9.239, "count": 123, "max_ms": 13.26, "p95_ms": 11.02}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.82, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 209}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4199.816, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5561213e3f80] error while decoding MB 41 44, bytestream -50", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:21:26+09:00", "last_error_summary": "[h264 @ 0x5561213e3f80] error while decoding MB 41 44, bytestream -50", "last_exit_at": "2026-07-01T16:19:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.798, "last_started_at": "2026-07-01T16:19:38+09:00", "pid": 8047, "process": {"exists": false, "pid": 8047}, "restart_count": 44, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.234, "sample_count": 1336, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25198, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:21:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T16:21:25+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T16:21:26+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T16:21:26.019950+09:00", "mtime_epoch": 1782890486.01995, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204532}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:21:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 869.355, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.326, "pushed": 4.326, "read": 4.326}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.427, "count": 129, "max_ms": 14.948, "p95_ms": 10.112}, "gstreamer_push": {"avg_ms": 0.923, "count": 129, "max_ms": 2.993, "p95_ms": 1.067}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.255, "count": 129, "max_ms": 70.604, "p95_ms": 68.392}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.237, "count": 129, "max_ms": 93.549, "p95_ms": 90.434}, "yolo11_640_fire_11class": {"avg_ms": 58.045, "count": 129, "max_ms": 79.798, "p95_ms": 72.941}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.829, "count": 129, "max_ms": 99.949, "p95_ms": 92.814}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.718, "count": 129, "max_ms": 308.682, "p95_ms": 254.792}, "read_interval": {"avg_ms": 232.383, "count": 129, "max_ms": 320.849, "p95_ms": 265.704}, "triton_wait": {"avg_ms": 74.133, "count": 129, "max_ms": 99.962, "p95_ms": 92.827}, "visualize": {"avg_ms": 10.147, "count": 129, "max_ms": 14.143, "p95_ms": 12.074}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.818, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 209}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4199.039, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:21:26+09:00", "last_error_summary": "[h264 @ 0x5561213e3f80] error while decoding MB 41 44, bytestream -50", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:21:26+09:00", "last_error_summary": "[h264 @ 0x5561213e3f80] error while decoding MB 41 44, bytestream -50", "last_exit_at": "2026-07-01T16:21:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.966, "last_started_at": "2026-07-01T16:19:38+09:00", "pid": 8047, "process": {"exists": false, "pid": 8047}, "restart_count": 44, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.234, "sample_count": 1336, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25199, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:21:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T16:21:26+09:00", "last_frame_read_age_sec": 0.22, "last_frame_read_at": "2026-07-01T16:21:26+09:00", "mjpeg": {"age_sec": 0.205, "exists": true, "mtime": "2026-07-01T16:21:26.019950+09:00", "mtime_epoch": 1782890486.01995, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204532}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:21:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 869.355, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.335, "pushed": 4.335, "read": 4.302}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.43, "count": 129, "max_ms": 14.948, "p95_ms": 10.112}, "gstreamer_push": {"avg_ms": 0.922, "count": 130, "max_ms": 2.993, "p95_ms": 1.067}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.261, "count": 129, "max_ms": 70.604, "p95_ms": 68.392}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.247, "count": 129, "max_ms": 93.549, "p95_ms": 90.434}, "yolo11_640_fire_11class": {"avg_ms": 58.05, "count": 129, "max_ms": 79.798, "p95_ms": 72.941}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.828, "count": 129, "max_ms": 99.949, "p95_ms": 92.814}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.656, "count": 130, "max_ms": 308.682, "p95_ms": 254.792}, "read_interval": {"avg_ms": 232.383, "count": 129, "max_ms": 320.849, "p95_ms": 265.704}, "triton_wait": {"avg_ms": 74.133, "count": 129, "max_ms": 99.962, "p95_ms": 92.827}, "visualize": {"avg_ms": 10.15, "count": 129, "max_ms": 14.143, "p95_ms": 12.074}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 209}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4199.027, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:21:26+09:00", "last_error_summary": "[h264 @ 0x5561213e3f80] error while decoding MB 41 44, bytestream -50", "last_exit_at": "2026-07-01T16:21:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.969, "last_started_at": "2026-07-01T16:19:38+09:00", "pid": 8047, "process": {"exists": false, "pid": 8047}, "restart_count": 44, "restart_count_per_hour": 27, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.234, "sample_count": 1336, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25199, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:21:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T16:21:26+09:00", "last_frame_read_age_sec": 0.224, "last_frame_read_at": "2026-07-01T16:21:26+09:00", "mjpeg": {"age_sec": 0.209, "exists": true, "mtime": "2026-07-01T16:21:26.019950+09:00", "mtime_epoch": 1782890486.01995, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204532}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:21:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 869.355, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.335, "pushed": 4.335, "read": 4.302}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.43, "count": 129, "max_ms": 14.948, "p95_ms": 10.112}, "gstreamer_push": {"avg_ms": 0.922, "count": 130, "max_ms": 2.993, "p95_ms": 1.067}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.261, "count": 129, "max_ms": 70.604, "p95_ms": 68.392}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.247, "count": 129, "max_ms": 93.549, "p95_ms": 90.434}, "yolo11_640_fire_11class": {"avg_ms": 58.05, "count": 129, "max_ms": 79.798, "p95_ms": 72.941}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.828, "count": 129, "max_ms": 99.949, "p95_ms": 92.814}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.656, "count": 130, "max_ms": 308.682, "p95_ms": 254.792}, "read_interval": {"avg_ms": 232.383, "count": 129, "max_ms": 320.849, "p95_ms": 265.704}, "triton_wait": {"avg_ms": 74.133, "count": 129, "max_ms": 99.962, "p95_ms": 92.827}, "visualize": {"avg_ms": 10.15, "count": 129, "max_ms": 14.143, "p95_ms": 12.074}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 209}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4311.641, "limit_mb": 8796093022207.996}, "detail": {"pid": 8154, "restart_count": 45, "start_count": 46}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:21:26+09:00", "last_error_summary": "[h264 @ 0x5561213e3f80] error while decoding MB 41 44, bytestream -50", "last_exit_at": "2026-07-01T16:21:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:21:28+09:00", "pid": 8154, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.434, "name": "ffmpeg", "pid": 8154, "rss_mb": 144.434, "state": "S (sleeping)", "threads": 106}, "restart_count": 45, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.349, "sample_count": 1313, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25199, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:21:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.408, "last_frame_pushed_at": "2026-07-01T16:21:26+09:00", "last_frame_read_age_sec": 4.625, "last_frame_read_at": "2026-07-01T16:21:26+09:00", "mjpeg": {"age_sec": 4.382, "exists": true, "mtime": "2026-07-01T16:21:26.247954+09:00", "mtime_epoch": 1782890486.247954, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198090}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:21:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 869.355, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.718, "pushed": 3.718, "read": 3.684}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.392, "count": 110, "max_ms": 14.948, "p95_ms": 10.132}, "gstreamer_push": {"avg_ms": 0.925, "count": 111, "max_ms": 2.993, "p95_ms": 1.067}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.528, "count": 110, "max_ms": 70.604, "p95_ms": 68.653}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.057, "count": 110, "max_ms": 93.549, "p95_ms": 90.564}, "yolo11_640_fire_11class": {"avg_ms": 58.166, "count": 110, "max_ms": 79.798, "p95_ms": 73.136}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.64, "count": 110, "max_ms": 99.949, "p95_ms": 92.814}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.016, "count": 111, "max_ms": 308.682, "p95_ms": 254.644}, "read_interval": {"avg_ms": 231.796, "count": 110, "max_ms": 320.849, "p95_ms": 265.704}, "triton_wait": {"avg_ms": 73.97, "count": 110, "max_ms": 99.962, "p95_ms": 92.827}, "visualize": {"avg_ms": 10.101, "count": 110, "max_ms": 14.143, "p95_ms": 12.187}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.856, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 425}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4070.633, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562f14882e00] error while decoding MB 114 3, bytestream -17", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:21:39+09:00", "last_error_summary": "[h264 @ 0x562f14882e00] error while decoding MB 114 3, bytestream -17", "last_exit_at": "2026-07-01T16:03:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 1071.585, "last_started_at": "2026-07-01T16:03:48+09:00", "pid": 7086, "process": {"exists": false, "pid": 7086}, "restart_count": 5, "restart_count_per_hour": 2, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.997, "sample_count": 2135, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51012, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:21:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-01T16:21:39+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T16:21:39+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-01T16:21:39.904186+09:00", "mtime_epoch": 1782890499.9041855, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 263645}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:21:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 1114.719, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 335, "pushed_frames": 335, "read_frames": 335, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 11.174, "pushed": 11.174, "read": 11.174}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.223, "count": 335, "max_ms": 4.875, "p95_ms": 3.948}, "gstreamer_push": {"avg_ms": 1.308, "count": 335, "max_ms": 1.535, "p95_ms": 1.421}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.526, "count": 335, "max_ms": 72.832, "p95_ms": 60.773}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.019, "count": 335, "max_ms": 96.247, "p95_ms": 88.312}, "yolo11_640_fire_11class": {"avg_ms": 55.32, "count": 335, "max_ms": 81.458, "p95_ms": 71.196}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.065, "count": 335, "max_ms": 97.063, "p95_ms": 91.39}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.325, "count": 335, "max_ms": 109.09, "p95_ms": 103.531}, "read_interval": {"avg_ms": 89.526, "count": 335, "max_ms": 127.488, "p95_ms": 120.487}, "triton_wait": {"avg_ms": 61.466, "count": 335, "max_ms": 97.076, "p95_ms": 91.73}, "visualize": {"avg_ms": 3.63, "count": 335, "max_ms": 5.887, "p95_ms": 3.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 425}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4008.387, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:21:39+09:00", "last_error_summary": "[h264 @ 0x562f14882e00] error while decoding MB 114 3, bytestream -17", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:21:39+09:00", "last_error_summary": "[h264 @ 0x562f14882e00] error while decoding MB 114 3, bytestream -17", "last_exit_at": "2026-07-01T16:21:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 1071.617, "last_started_at": "2026-07-01T16:03:48+09:00", "pid": 7086, "process": {"exists": false, "pid": 7086}, "restart_count": 5, "restart_count_per_hour": 2, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.997, "sample_count": 2136, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51013, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:21:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-01T16:21:39+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-01T16:21:39+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-01T16:21:39.904186+09:00", "mtime_epoch": 1782890499.9041855, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 263645}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:21:40+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 1051.793, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 335, "pushed_frames": 335, "read_frames": 335, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 11.167, "pushed": 11.167, "read": 11.167}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.223, "count": 335, "max_ms": 4.875, "p95_ms": 3.948}, "gstreamer_push": {"avg_ms": 1.308, "count": 335, "max_ms": 1.535, "p95_ms": 1.421}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.511, "count": 335, "max_ms": 72.832, "p95_ms": 60.773}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.011, "count": 335, "max_ms": 96.247, "p95_ms": 88.312}, "yolo11_640_fire_11class": {"avg_ms": 55.311, "count": 335, "max_ms": 81.458, "p95_ms": 71.196}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.055, "count": 335, "max_ms": 97.063, "p95_ms": 91.39}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.335, "count": 335, "max_ms": 109.09, "p95_ms": 103.531}, "read_interval": {"avg_ms": 89.526, "count": 335, "max_ms": 127.488, "p95_ms": 120.487}, "triton_wait": {"avg_ms": 61.457, "count": 335, "max_ms": 97.076, "p95_ms": 91.73}, "visualize": {"avg_ms": 3.645, "count": 335, "max_ms": 8.703, "p95_ms": 3.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 425}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4005.973, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:21:39+09:00", "last_error_summary": "[h264 @ 0x562f14882e00] error while decoding MB 114 3, bytestream -17", "last_exit_at": "2026-07-01T16:21:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 1071.626, "last_started_at": "2026-07-01T16:03:48+09:00", "pid": 7086, "process": {"exists": false, "pid": 7086}, "restart_count": 5, "restart_count_per_hour": 2, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.997, "sample_count": 2136, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51013, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:21:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.027, "last_frame_pushed_at": "2026-07-01T16:21:39+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-01T16:21:39+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-01T16:21:39.904186+09:00", "mtime_epoch": 1782890499.9041855, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 263645}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:21:40+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 1051.793, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 335, "pushed_frames": 335, "read_frames": 334, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 11.189, "pushed": 11.189, "read": 11.155}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.223, "count": 335, "max_ms": 4.875, "p95_ms": 3.948}, "gstreamer_push": {"avg_ms": 1.308, "count": 335, "max_ms": 1.535, "p95_ms": 1.421}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.511, "count": 335, "max_ms": 72.832, "p95_ms": 60.773}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.011, "count": 335, "max_ms": 96.247, "p95_ms": 88.312}, "yolo11_640_fire_11class": {"avg_ms": 55.311, "count": 335, "max_ms": 81.458, "p95_ms": 71.196}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.055, "count": 335, "max_ms": 97.063, "p95_ms": 91.39}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.335, "count": 335, "max_ms": 109.09, "p95_ms": 103.531}, "read_interval": {"avg_ms": 89.526, "count": 334, "max_ms": 127.488, "p95_ms": 120.487}, "triton_wait": {"avg_ms": 61.457, "count": 335, "max_ms": 97.076, "p95_ms": 91.73}, "visualize": {"avg_ms": 3.645, "count": 335, "max_ms": 8.703, "p95_ms": 3.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 425}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4222.719, "limit_mb": 8796093022207.996}, "detail": {"pid": 8261, "restart_count": 6, "start_count": 7}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:21:39+09:00", "last_error_summary": "[h264 @ 0x562f14882e00] error while decoding MB 114 3, bytestream -17", "last_exit_at": "2026-07-01T16:21:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:21:42+09:00", "pid": 8261, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.758, "name": "ffmpeg", "pid": 8261, "rss_mb": 209.758, "state": "S (sleeping)", "threads": 106}, "restart_count": 6, "restart_count_per_hour": 3, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 176.726, "sample_count": 2109, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51013, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:21:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.912, "last_frame_pushed_at": "2026-07-01T16:21:39+09:00", "last_frame_read_age_sec": 4.991, "last_frame_read_at": "2026-07-01T16:21:39+09:00", "mjpeg": {"age_sec": 4.891, "exists": true, "mtime": "2026-07-01T16:21:40.004187+09:00", "mtime_epoch": 1782890500.004187, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 263645}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:21:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 1051.793, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 281, "pushed_frames": 281, "read_frames": 280, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.381, "pushed": 9.381, "read": 9.348}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.24, "count": 281, "max_ms": 4.875, "p95_ms": 4.097}, "gstreamer_push": {"avg_ms": 1.309, "count": 281, "max_ms": 1.535, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.475, "count": 281, "max_ms": 72.832, "p95_ms": 60.773}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.934, "count": 281, "max_ms": 96.247, "p95_ms": 88.312}, "yolo11_640_fire_11class": {"avg_ms": 55.247, "count": 281, "max_ms": 81.458, "p95_ms": 72.349}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.014, "count": 281, "max_ms": 97.063, "p95_ms": 91.39}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.308, "count": 281, "max_ms": 109.09, "p95_ms": 103.805}, "read_interval": {"avg_ms": 89.423, "count": 280, "max_ms": 127.488, "p95_ms": 120.706}, "triton_wait": {"avg_ms": 61.405, "count": 281, "max_ms": 97.076, "p95_ms": 91.73}, "visualize": {"avg_ms": 3.651, "count": 281, "max_ms": 8.703, "p95_ms": 3.972}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 213}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4086.371, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563de479b780] error while decoding MB 79 12, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:23:16+09:00", "last_error_summary": "[h264 @ 0x563de479b780] error while decoding MB 79 12, bytestream -9", "last_exit_at": "2026-07-01T16:21:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.968, "last_started_at": "2026-07-01T16:21:28+09:00", "pid": 8154, "process": {"exists": false, "pid": 8154}, "restart_count": 45, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.193, "sample_count": 1293, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:23:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T16:23:16+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T16:23:16+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T16:23:16.553826+09:00", "mtime_epoch": 1782890596.553826, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:23:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 867.695, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.268, "pushed": 4.268, "read": 4.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.759, "count": 128, "max_ms": 12.055, "p95_ms": 11.471}, "gstreamer_push": {"avg_ms": 0.942, "count": 128, "max_ms": 2.919, "p95_ms": 1.084}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.824, "count": 128, "max_ms": 70.355, "p95_ms": 63.505}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.547, "count": 128, "max_ms": 89.056, "p95_ms": 84.876}, "yolo11_640_fire_11class": {"avg_ms": 54.971, "count": 128, "max_ms": 74.372, "p95_ms": 71.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.577, "count": 128, "max_ms": 96.757, "p95_ms": 90.14}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.047, "count": 128, "max_ms": 298.383, "p95_ms": 257.999}, "read_interval": {"avg_ms": 233.862, "count": 129, "max_ms": 307.828, "p95_ms": 268.08}, "triton_wait": {"avg_ms": 69.733, "count": 128, "max_ms": 96.77, "p95_ms": 90.153}, "visualize": {"avg_ms": 10.617, "count": 128, "max_ms": 14.513, "p95_ms": 13.423}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 213}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4086.027, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:23:16+09:00", "last_error_summary": "[h264 @ 0x563de479b780] error while decoding MB 79 12, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:23:16+09:00", "last_error_summary": "[h264 @ 0x563de479b780] error while decoding MB 79 12, bytestream -9", "last_exit_at": "2026-07-01T16:23:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.136, "last_started_at": "2026-07-01T16:21:28+09:00", "pid": 8154, "process": {"exists": false, "pid": 8154}, "restart_count": 45, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.193, "sample_count": 1293, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25660, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:23:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:23:16+09:00", "last_frame_read_age_sec": 0.22, "last_frame_read_at": "2026-07-01T16:23:16+09:00", "mjpeg": {"age_sec": 0.211, "exists": true, "mtime": "2026-07-01T16:23:16.553826+09:00", "mtime_epoch": 1782890596.553826, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:23:16+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 867.695, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.309, "pushed": 4.309, "read": 4.276}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.755, "count": 128, "max_ms": 12.055, "p95_ms": 11.471}, "gstreamer_push": {"avg_ms": 0.942, "count": 129, "max_ms": 2.919, "p95_ms": 1.084}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.869, "count": 128, "max_ms": 70.355, "p95_ms": 63.505}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.596, "count": 128, "max_ms": 89.056, "p95_ms": 84.876}, "yolo11_640_fire_11class": {"avg_ms": 55.035, "count": 128, "max_ms": 74.372, "p95_ms": 71.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.625, "count": 128, "max_ms": 96.757, "p95_ms": 90.14}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.982, "count": 129, "max_ms": 298.383, "p95_ms": 257.999}, "read_interval": {"avg_ms": 233.901, "count": 128, "max_ms": 307.828, "p95_ms": 268.08}, "triton_wait": {"avg_ms": 69.782, "count": 128, "max_ms": 96.77, "p95_ms": 90.153}, "visualize": {"avg_ms": 10.618, "count": 128, "max_ms": 14.513, "p95_ms": 13.423}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 213}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4086.02, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:23:16+09:00", "last_error_summary": "[h264 @ 0x563de479b780] error while decoding MB 79 12, bytestream -9", "last_exit_at": "2026-07-01T16:23:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.139, "last_started_at": "2026-07-01T16:21:28+09:00", "pid": 8154, "process": {"exists": false, "pid": 8154}, "restart_count": 45, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.193, "sample_count": 1293, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25660, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:23:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:23:16+09:00", "last_frame_read_age_sec": 0.223, "last_frame_read_at": "2026-07-01T16:23:16+09:00", "mjpeg": {"age_sec": 0.214, "exists": true, "mtime": "2026-07-01T16:23:16.553826+09:00", "mtime_epoch": 1782890596.553826, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:23:16+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 867.695, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.309, "pushed": 4.309, "read": 4.276}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.755, "count": 128, "max_ms": 12.055, "p95_ms": 11.471}, "gstreamer_push": {"avg_ms": 0.942, "count": 129, "max_ms": 2.919, "p95_ms": 1.084}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.869, "count": 128, "max_ms": 70.355, "p95_ms": 63.505}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.596, "count": 128, "max_ms": 89.056, "p95_ms": 84.876}, "yolo11_640_fire_11class": {"avg_ms": 55.035, "count": 128, "max_ms": 74.372, "p95_ms": 71.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.625, "count": 128, "max_ms": 96.757, "p95_ms": 90.14}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.982, "count": 129, "max_ms": 298.383, "p95_ms": 257.999}, "read_interval": {"avg_ms": 233.901, "count": 128, "max_ms": 307.828, "p95_ms": 268.08}, "triton_wait": {"avg_ms": 69.782, "count": 128, "max_ms": 96.77, "p95_ms": 90.153}, "visualize": {"avg_ms": 10.618, "count": 128, "max_ms": 14.513, "p95_ms": 13.423}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 213}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4191.414, "limit_mb": 8796093022207.996}, "detail": {"pid": 8413, "restart_count": 46, "start_count": 47}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:23:16+09:00", "last_error_summary": "[h264 @ 0x563de479b780] error while decoding MB 79 12, bytestream -9", "last_exit_at": "2026-07-01T16:23:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:23:19+09:00", "pid": 8413, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.48, "name": "ffmpeg", "pid": 8413, "rss_mb": 136.48, "state": "S (sleeping)", "threads": 106}, "restart_count": 46, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.237, "sample_count": 1274, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 25660, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:23:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.512, "last_frame_pushed_at": "2026-07-01T16:23:16+09:00", "last_frame_read_age_sec": 4.729, "last_frame_read_at": "2026-07-01T16:23:16+09:00", "mjpeg": {"age_sec": 4.496, "exists": true, "mtime": "2026-07-01T16:23:16.777830+09:00", "mtime_epoch": 1782890596.77783, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196772}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:23:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 867.695, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.715, "pushed": 3.715, "read": 3.682}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.46, "count": 110, "max_ms": 11.489, "p95_ms": 9.488}, "gstreamer_push": {"avg_ms": 0.942, "count": 111, "max_ms": 2.919, "p95_ms": 1.084}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.219, "count": 110, "max_ms": 70.355, "p95_ms": 63.505}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.881, "count": 110, "max_ms": 89.056, "p95_ms": 87.037}, "yolo11_640_fire_11class": {"avg_ms": 55.269, "count": 110, "max_ms": 74.372, "p95_ms": 71.476}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.824, "count": 110, "max_ms": 96.757, "p95_ms": 90.14}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.089, "count": 111, "max_ms": 295.161, "p95_ms": 257.319}, "read_interval": {"avg_ms": 230.963, "count": 110, "max_ms": 304.404, "p95_ms": 267.953}, "triton_wait": {"avg_ms": 70.003, "count": 110, "max_ms": 96.77, "p95_ms": 90.153}, "visualize": {"avg_ms": 10.323, "count": 110, "max_ms": 13.264, "p95_ms": 11.34}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4090.211, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56118c9844c0] error while decoding MB 3 25, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:25:07+09:00", "last_error_summary": "[h264 @ 0x56118c9844c0] error while decoding MB 3 25, bytestream -7", "last_exit_at": "2026-07-01T16:23:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.857, "last_started_at": "2026-07-01T16:23:19+09:00", "pid": 8413, "process": {"exists": false, "pid": 8413}, "restart_count": 46, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.369, "sample_count": 1274, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26126, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:25:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T16:25:07+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T16:25:07+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T16:25:07.087704+09:00", "mtime_epoch": 1782890707.087704, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194527}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:25:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 870.719, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.371, "pushed": 4.371, "read": 4.371}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.222, "count": 130, "max_ms": 11.795, "p95_ms": 9.564}, "gstreamer_push": {"avg_ms": 0.984, "count": 131, "max_ms": 2.923, "p95_ms": 1.815}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.206, "count": 130, "max_ms": 71.118, "p95_ms": 62.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.562, "count": 130, "max_ms": 93.892, "p95_ms": 87.495}, "yolo11_640_fire_11class": {"avg_ms": 56.48, "count": 130, "max_ms": 76.379, "p95_ms": 71.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.955, "count": 130, "max_ms": 94.655, "p95_ms": 89.804}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.086, "count": 131, "max_ms": 316.351, "p95_ms": 239.016}, "read_interval": {"avg_ms": 229.912, "count": 131, "max_ms": 328.098, "p95_ms": 248.701}, "triton_wait": {"avg_ms": 71.315, "count": 130, "max_ms": 94.667, "p95_ms": 89.935}, "visualize": {"avg_ms": 10.22, "count": 130, "max_ms": 15.189, "p95_ms": 12.347}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4089.289, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:25:07+09:00", "last_error_summary": "[h264 @ 0x56118c9844c0] error while decoding MB 3 25, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:25:07+09:00", "last_error_summary": "[h264 @ 0x56118c9844c0] error while decoding MB 3 25, bytestream -7", "last_exit_at": "2026-07-01T16:25:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.028, "last_started_at": "2026-07-01T16:23:19+09:00", "pid": 8413, "process": {"exists": false, "pid": 8413}, "restart_count": 46, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.369, "sample_count": 1275, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26127, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:25:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:25:07+09:00", "last_frame_read_age_sec": 0.223, "last_frame_read_at": "2026-07-01T16:25:07+09:00", "mjpeg": {"age_sec": 0.213, "exists": true, "mtime": "2026-07-01T16:25:07.087704+09:00", "mtime_epoch": 1782890707.087704, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194527}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:25:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 870.719, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.384, "pushed": 4.384, "read": 4.351}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.224, "count": 130, "max_ms": 11.795, "p95_ms": 9.564}, "gstreamer_push": {"avg_ms": 0.983, "count": 131, "max_ms": 2.923, "p95_ms": 1.815}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.352, "count": 130, "max_ms": 71.118, "p95_ms": 62.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.743, "count": 130, "max_ms": 93.892, "p95_ms": 87.495}, "yolo11_640_fire_11class": {"avg_ms": 56.633, "count": 130, "max_ms": 76.379, "p95_ms": 71.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.108, "count": 130, "max_ms": 94.655, "p95_ms": 89.804}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.199, "count": 131, "max_ms": 316.351, "p95_ms": 239.016}, "read_interval": {"avg_ms": 230.045, "count": 130, "max_ms": 328.098, "p95_ms": 248.701}, "triton_wait": {"avg_ms": 71.468, "count": 130, "max_ms": 94.667, "p95_ms": 89.935}, "visualize": {"avg_ms": 10.218, "count": 130, "max_ms": 15.189, "p95_ms": 12.347}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4089.281, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:25:07+09:00", "last_error_summary": "[h264 @ 0x56118c9844c0] error while decoding MB 3 25, bytestream -7", "last_exit_at": "2026-07-01T16:25:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.032, "last_started_at": "2026-07-01T16:23:19+09:00", "pid": 8413, "process": {"exists": false, "pid": 8413}, "restart_count": 46, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.369, "sample_count": 1275, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26127, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:25:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:25:07+09:00", "last_frame_read_age_sec": 0.226, "last_frame_read_at": "2026-07-01T16:25:07+09:00", "mjpeg": {"age_sec": 0.216, "exists": true, "mtime": "2026-07-01T16:25:07.087704+09:00", "mtime_epoch": 1782890707.087704, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194527}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:25:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 870.719, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.384, "pushed": 4.384, "read": 4.351}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.224, "count": 130, "max_ms": 11.795, "p95_ms": 9.564}, "gstreamer_push": {"avg_ms": 0.983, "count": 131, "max_ms": 2.923, "p95_ms": 1.815}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.352, "count": 130, "max_ms": 71.118, "p95_ms": 62.708}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.743, "count": 130, "max_ms": 93.892, "p95_ms": 87.495}, "yolo11_640_fire_11class": {"avg_ms": 56.633, "count": 130, "max_ms": 76.379, "p95_ms": 71.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.108, "count": 130, "max_ms": 94.655, "p95_ms": 89.804}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.199, "count": 131, "max_ms": 316.351, "p95_ms": 239.016}, "read_interval": {"avg_ms": 230.045, "count": 130, "max_ms": 328.098, "p95_ms": 248.701}, "triton_wait": {"avg_ms": 71.468, "count": 130, "max_ms": 94.667, "p95_ms": 89.935}, "visualize": {"avg_ms": 10.218, "count": 130, "max_ms": 15.189, "p95_ms": 12.347}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 217}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4194.68, "limit_mb": 8796093022207.996}, "detail": {"pid": 8520, "restart_count": 47, "start_count": 48}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:25:07+09:00", "last_error_summary": "[h264 @ 0x56118c9844c0] error while decoding MB 3 25, bytestream -7", "last_exit_at": "2026-07-01T16:25:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:25:10+09:00", "pid": 8520, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.695, "name": "ffmpeg", "pid": 8520, "rss_mb": 136.695, "state": "S (sleeping)", "threads": 106}, "restart_count": 47, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.039, "sample_count": 1253, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26127, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:25:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.966, "last_frame_pushed_at": "2026-07-01T16:25:07+09:00", "last_frame_read_age_sec": 5.186, "last_frame_read_at": "2026-07-01T16:25:07+09:00", "mjpeg": {"age_sec": 4.948, "exists": true, "mtime": "2026-07-01T16:25:07.315708+09:00", "mtime_epoch": 1782890707.315708, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194484}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:25:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 870.719, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.651, "pushed": 3.651, "read": 3.617}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.158, "count": 109, "max_ms": 10.989, "p95_ms": 9.359}, "gstreamer_push": {"avg_ms": 0.978, "count": 109, "max_ms": 2.923, "p95_ms": 1.815}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.261, "count": 109, "max_ms": 71.118, "p95_ms": 62.971}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.761, "count": 109, "max_ms": 93.892, "p95_ms": 87.495}, "yolo11_640_fire_11class": {"avg_ms": 56.802, "count": 109, "max_ms": 76.379, "p95_ms": 71.317}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.166, "count": 109, "max_ms": 94.655, "p95_ms": 89.974}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.532, "count": 109, "max_ms": 316.351, "p95_ms": 239.016}, "read_interval": {"avg_ms": 230.335, "count": 108, "max_ms": 328.098, "p95_ms": 248.701}, "triton_wait": {"avg_ms": 71.559, "count": 109, "max_ms": 94.667, "p95_ms": 89.986}, "visualize": {"avg_ms": 10.149, "count": 109, "max_ms": 15.189, "p95_ms": 11.454}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.855, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 221}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4093.262, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:25:07+09:00", "last_error_summary": "[h264 @ 0x56118c9844c0] error while decoding MB 3 25, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:25:07+09:00", "last_error_summary": "[h264 @ 0x56118c9844c0] error while decoding MB 3 25, bytestream -7", "last_exit_at": "2026-07-01T16:26:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.518, "last_started_at": "2026-07-01T16:25:10+09:00", "pid": 8520, "process": {"exists": false, "pid": 8520}, "restart_count": 47, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.348, "sample_count": 1272, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26596, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:26:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:26:58+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T16:26:58+09:00", "mjpeg": {"age_sec": 0.231, "exists": true, "mtime": "2026-07-01T16:26:58.549599+09:00", "mtime_epoch": 1782890818.5495994, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196235}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:26:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 873.301, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.462, "pushed": 4.462, "read": 4.428}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.053, "count": 133, "max_ms": 12.081, "p95_ms": 8.709}, "gstreamer_push": {"avg_ms": 0.933, "count": 133, "max_ms": 2.915, "p95_ms": 1.772}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.486, "count": 133, "max_ms": 75.932, "p95_ms": 64.97}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.067, "count": 133, "max_ms": 94.053, "p95_ms": 87.659}, "yolo11_640_fire_11class": {"avg_ms": 56.955, "count": 133, "max_ms": 80.816, "p95_ms": 74.969}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.262, "count": 133, "max_ms": 100.354, "p95_ms": 92.536}}, "truncated_count": 0}, "processing_total": {"avg_ms": 216.214, "count": 133, "max_ms": 276.042, "p95_ms": 245.539}, "read_interval": {"avg_ms": 225.587, "count": 132, "max_ms": 286.003, "p95_ms": 254.753}, "triton_wait": {"avg_ms": 71.399, "count": 133, "max_ms": 100.365, "p95_ms": 92.55}, "visualize": {"avg_ms": 9.873, "count": 133, "max_ms": 12.731, "p95_ms": 10.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.809, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 221}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4093.254, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:25:07+09:00", "last_error_summary": "[h264 @ 0x56118c9844c0] error while decoding MB 3 25, bytestream -7", "last_exit_at": "2026-07-01T16:26:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.521, "last_started_at": "2026-07-01T16:25:10+09:00", "pid": 8520, "process": {"exists": false, "pid": 8520}, "restart_count": 47, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.348, "sample_count": 1272, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26596, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:26:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T16:26:58+09:00", "last_frame_read_age_sec": 0.241, "last_frame_read_at": "2026-07-01T16:26:58+09:00", "mjpeg": {"age_sec": 0.235, "exists": true, "mtime": "2026-07-01T16:26:58.549599+09:00", "mtime_epoch": 1782890818.5495994, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196235}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:26:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 873.301, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.461, "pushed": 4.461, "read": 4.428}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.053, "count": 133, "max_ms": 12.081, "p95_ms": 8.709}, "gstreamer_push": {"avg_ms": 0.933, "count": 133, "max_ms": 2.915, "p95_ms": 1.772}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.486, "count": 133, "max_ms": 75.932, "p95_ms": 64.97}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.067, "count": 133, "max_ms": 94.053, "p95_ms": 87.659}, "yolo11_640_fire_11class": {"avg_ms": 56.955, "count": 133, "max_ms": 80.816, "p95_ms": 74.969}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.262, "count": 133, "max_ms": 100.354, "p95_ms": 92.536}}, "truncated_count": 0}, "processing_total": {"avg_ms": 216.214, "count": 133, "max_ms": 276.042, "p95_ms": 245.539}, "read_interval": {"avg_ms": 225.587, "count": 132, "max_ms": 286.003, "p95_ms": 254.753}, "triton_wait": {"avg_ms": 71.399, "count": 133, "max_ms": 100.365, "p95_ms": 92.55}, "visualize": {"avg_ms": 9.873, "count": 133, "max_ms": 12.731, "p95_ms": 10.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.812, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 221}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4201.336, "limit_mb": 8796093022207.996}, "detail": {"pid": 8627, "restart_count": 48, "start_count": 49}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:25:07+09:00", "last_error_summary": "[h264 @ 0x56118c9844c0] error while decoding MB 3 25, bytestream -7", "last_exit_at": "2026-07-01T16:27:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:27:01+09:00", "pid": 8627, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.105, "name": "ffmpeg", "pid": 8627, "rss_mb": 139.105, "state": "S (sleeping)", "threads": 106}, "restart_count": 48, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.539, "sample_count": 1250, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 26596, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:26:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.974, "last_frame_pushed_at": "2026-07-01T16:26:58+09:00", "last_frame_read_age_sec": 5.209, "last_frame_read_at": "2026-07-01T16:26:58+09:00", "mjpeg": {"age_sec": 4.958, "exists": true, "mtime": "2026-07-01T16:26:58.793603+09:00", "mtime_epoch": 1782890818.7936034, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196623}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:27:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 873.301, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.721, "pushed": 3.721, "read": 3.687}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.059, "count": 111, "max_ms": 12.081, "p95_ms": 8.709}, "gstreamer_push": {"avg_ms": 0.949, "count": 111, "max_ms": 2.915, "p95_ms": 1.781}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.014, "count": 111, "max_ms": 75.932, "p95_ms": 66.13}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.549, "count": 111, "max_ms": 94.053, "p95_ms": 88.046}, "yolo11_640_fire_11class": {"avg_ms": 56.671, "count": 111, "max_ms": 80.816, "p95_ms": 74.969}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.958, "count": 111, "max_ms": 100.354, "p95_ms": 92.856}}, "truncated_count": 0}, "processing_total": {"avg_ms": 216.562, "count": 111, "max_ms": 276.042, "p95_ms": 250.457}, "read_interval": {"avg_ms": 225.937, "count": 110, "max_ms": 286.003, "p95_ms": 260.055}, "triton_wait": {"avg_ms": 71.112, "count": 111, "max_ms": 100.365, "p95_ms": 92.869}, "visualize": {"avg_ms": 9.884, "count": 111, "max_ms": 12.731, "p95_ms": 10.803}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.833, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3955.07, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55afa9cb4ec0] error while decoding MB 46 22, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:29:12+09:00", "last_error_summary": "[h264 @ 0x55afa9cb4ec0] error while decoding MB 46 22, bytestream -7", "last_exit_at": "2026-07-01T16:27:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.256, "last_started_at": "2026-07-01T16:27:01+09:00", "pid": 8627, "process": {"exists": false, "pid": 8627}, "restart_count": 48, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 239.253, "sample_count": 1275, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27159, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:29:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T16:29:11+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T16:29:11+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T16:29:11.963870+09:00", "mtime_epoch": 1782890951.9638696, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192422}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:29:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 871.172, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.398, "pushed": 4.398, "read": 4.398}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.14, "count": 131, "max_ms": 9.679, "p95_ms": 9.219}, "gstreamer_push": {"avg_ms": 0.916, "count": 131, "max_ms": 2.926, "p95_ms": 0.969}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.717, "count": 131, "max_ms": 71.7, "p95_ms": 69.112}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.627, "count": 131, "max_ms": 91.651, "p95_ms": 88.692}, "yolo11_640_fire_11class": {"avg_ms": 57.158, "count": 131, "max_ms": 73.197, "p95_ms": 71.694}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.081, "count": 131, "max_ms": 94.627, "p95_ms": 92.947}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.89, "count": 131, "max_ms": 285.441, "p95_ms": 238.031}, "read_interval": {"avg_ms": 228.683, "count": 131, "max_ms": 295.71, "p95_ms": 247.671}, "triton_wait": {"avg_ms": 73.34, "count": 131, "max_ms": 94.64, "p95_ms": 92.957}, "visualize": {"avg_ms": 10.015, "count": 131, "max_ms": 11.987, "p95_ms": 11.034}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.784, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3954.102, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:29:12+09:00", "last_error_summary": "[h264 @ 0x55afa9cb4ec0] error while decoding MB 46 22, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:29:12+09:00", "last_error_summary": "[h264 @ 0x55afa9cb4ec0] error while decoding MB 46 22, bytestream -7", "last_exit_at": "2026-07-01T16:29:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.452, "last_started_at": "2026-07-01T16:27:01+09:00", "pid": 8627, "process": {"exists": false, "pid": 8627}, "restart_count": 48, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 239.337, "sample_count": 1275, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27160, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:29:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:29:12+09:00", "last_frame_read_age_sec": 0.25, "last_frame_read_at": "2026-07-01T16:29:11+09:00", "mjpeg": {"age_sec": 0.239, "exists": true, "mtime": "2026-07-01T16:29:11.963870+09:00", "mtime_epoch": 1782890951.9638696, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192422}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:29:12+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 871.172, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.403, "pushed": 4.403, "read": 4.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.149, "count": 131, "max_ms": 9.679, "p95_ms": 9.219}, "gstreamer_push": {"avg_ms": 0.915, "count": 132, "max_ms": 2.926, "p95_ms": 0.969}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.691, "count": 131, "max_ms": 71.7, "p95_ms": 69.112}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.578, "count": 131, "max_ms": 91.651, "p95_ms": 88.692}, "yolo11_640_fire_11class": {"avg_ms": 57.131, "count": 131, "max_ms": 73.197, "p95_ms": 71.694}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.001, "count": 131, "max_ms": 94.627, "p95_ms": 92.947}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.083, "count": 132, "max_ms": 285.441, "p95_ms": 245.38}, "read_interval": {"avg_ms": 228.683, "count": 131, "max_ms": 295.71, "p95_ms": 247.671}, "triton_wait": {"avg_ms": 73.273, "count": 131, "max_ms": 94.64, "p95_ms": 92.957}, "visualize": {"avg_ms": 10.026, "count": 131, "max_ms": 11.987, "p95_ms": 11.034}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3954.094, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:29:12+09:00", "last_error_summary": "[h264 @ 0x55afa9cb4ec0] error while decoding MB 46 22, bytestream -7", "last_exit_at": "2026-07-01T16:29:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.456, "last_started_at": "2026-07-01T16:27:01+09:00", "pid": 8627, "process": {"exists": false, "pid": 8627}, "restart_count": 48, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 239.337, "sample_count": 1275, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27160, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:29:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T16:29:12+09:00", "last_frame_read_age_sec": 0.253, "last_frame_read_at": "2026-07-01T16:29:11+09:00", "mjpeg": {"age_sec": 0.242, "exists": true, "mtime": "2026-07-01T16:29:11.963870+09:00", "mtime_epoch": 1782890951.9638696, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192422}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:29:12+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 871.172, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.402, "pushed": 4.402, "read": 4.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.149, "count": 131, "max_ms": 9.679, "p95_ms": 9.219}, "gstreamer_push": {"avg_ms": 0.915, "count": 132, "max_ms": 2.926, "p95_ms": 0.969}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.691, "count": 131, "max_ms": 71.7, "p95_ms": 69.112}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.578, "count": 131, "max_ms": 91.651, "p95_ms": 88.692}, "yolo11_640_fire_11class": {"avg_ms": 57.131, "count": 131, "max_ms": 73.197, "p95_ms": 71.694}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.001, "count": 131, "max_ms": 94.627, "p95_ms": 92.947}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.083, "count": 132, "max_ms": 285.441, "p95_ms": 245.38}, "read_interval": {"avg_ms": 228.683, "count": 131, "max_ms": 295.71, "p95_ms": 247.671}, "triton_wait": {"avg_ms": 73.273, "count": 131, "max_ms": 94.64, "p95_ms": 92.957}, "visualize": {"avg_ms": 10.026, "count": 131, "max_ms": 11.987, "p95_ms": 11.034}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4065.27, "limit_mb": 8796093022207.996}, "detail": {"pid": 8734, "restart_count": 49, "start_count": 50}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:29:12+09:00", "last_error_summary": "[h264 @ 0x55afa9cb4ec0] error while decoding MB 46 22, bytestream -7", "last_exit_at": "2026-07-01T16:29:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:29:14+09:00", "pid": 8734, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.156, "name": "ffmpeg", "pid": 8734, "rss_mb": 139.156, "state": "S (sleeping)", "threads": 106}, "restart_count": 49, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 239.623, "sample_count": 1256, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27160, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:29:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.179, "last_frame_pushed_at": "2026-07-01T16:29:12+09:00", "last_frame_read_age_sec": 4.426, "last_frame_read_at": "2026-07-01T16:29:11+09:00", "mjpeg": {"age_sec": 4.159, "exists": true, "mtime": "2026-07-01T16:29:12.219874+09:00", "mtime_epoch": 1782890952.219874, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192858}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:29:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 871.172, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.802, "pushed": 3.802, "read": 3.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.218, "count": 113, "max_ms": 9.679, "p95_ms": 9.24}, "gstreamer_push": {"avg_ms": 0.905, "count": 114, "max_ms": 2.926, "p95_ms": 0.931}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.373, "count": 113, "max_ms": 71.7, "p95_ms": 69.112}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.097, "count": 113, "max_ms": 91.651, "p95_ms": 88.858}, "yolo11_640_fire_11class": {"avg_ms": 56.926, "count": 113, "max_ms": 73.197, "p95_ms": 71.906}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.589, "count": 113, "max_ms": 94.627, "p95_ms": 93.062}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.765, "count": 114, "max_ms": 269.117, "p95_ms": 245.38}, "read_interval": {"avg_ms": 228.182, "count": 113, "max_ms": 281.96, "p95_ms": 247.671}, "triton_wait": {"avg_ms": 72.804, "count": 113, "max_ms": 94.64, "p95_ms": 93.076}, "visualize": {"avg_ms": 10.064, "count": 113, "max_ms": 11.987, "p95_ms": 11.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 230}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3961.246, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555c5ad93c80] error while decoding MB 63 6, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:31:04+09:00", "last_error_summary": "[h264 @ 0x555c5ad93c80] error while decoding MB 63 6, bytestream -11", "last_exit_at": "2026-07-01T16:29:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.517, "last_started_at": "2026-07-01T16:29:14+09:00", "pid": 8734, "process": {"exists": false, "pid": 8734}, "restart_count": 49, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.564, "sample_count": 1275, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27632, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:31:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T16:31:04+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T16:31:04+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T16:31:04.853792+09:00", "mtime_epoch": 1782891064.8537917, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200705}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:31:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 873.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.356, "pushed": 4.356, "read": 4.356}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.977, "count": 130, "max_ms": 9.467, "p95_ms": 8.759}, "gstreamer_push": {"avg_ms": 0.928, "count": 130, "max_ms": 2.976, "p95_ms": 1.701}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.488, "count": 130, "max_ms": 71.99, "p95_ms": 66.642}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.627, "count": 130, "max_ms": 90.168, "p95_ms": 86.184}, "yolo11_640_fire_11class": {"avg_ms": 58.388, "count": 130, "max_ms": 78.182, "p95_ms": 73.105}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.859, "count": 130, "max_ms": 95.221, "p95_ms": 92.456}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.146, "count": 130, "max_ms": 289.789, "p95_ms": 256.738}, "read_interval": {"avg_ms": 230.814, "count": 130, "max_ms": 299.974, "p95_ms": 266.398}, "triton_wait": {"avg_ms": 74.103, "count": 130, "max_ms": 95.233, "p95_ms": 92.469}, "visualize": {"avg_ms": 9.869, "count": 130, "max_ms": 12.215, "p95_ms": 10.971}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 230}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3960.383, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:31:04+09:00", "last_error_summary": "[h264 @ 0x555c5ad93c80] error while decoding MB 63 6, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:31:04+09:00", "last_error_summary": "[h264 @ 0x555c5ad93c80] error while decoding MB 63 6, bytestream -11", "last_exit_at": "2026-07-01T16:31:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.697, "last_started_at": "2026-07-01T16:29:14+09:00", "pid": 8734, "process": {"exists": false, "pid": 8734}, "restart_count": 49, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.564, "sample_count": 1275, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27633, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:31:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T16:31:05+09:00", "last_frame_read_age_sec": 0.229, "last_frame_read_at": "2026-07-01T16:31:04+09:00", "mjpeg": {"age_sec": 0.221, "exists": true, "mtime": "2026-07-01T16:31:04.853792+09:00", "mtime_epoch": 1782891064.8537917, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200705}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:31:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 873.828, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.365, "pushed": 4.365, "read": 4.331}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.986, "count": 130, "max_ms": 9.467, "p95_ms": 8.759}, "gstreamer_push": {"avg_ms": 0.928, "count": 130, "max_ms": 2.976, "p95_ms": 1.701}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.459, "count": 130, "max_ms": 71.99, "p95_ms": 66.642}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.59, "count": 130, "max_ms": 90.168, "p95_ms": 86.184}, "yolo11_640_fire_11class": {"avg_ms": 58.345, "count": 130, "max_ms": 78.182, "p95_ms": 73.105}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.824, "count": 130, "max_ms": 95.221, "p95_ms": 92.456}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.248, "count": 130, "max_ms": 289.789, "p95_ms": 256.738}, "read_interval": {"avg_ms": 230.895, "count": 129, "max_ms": 299.974, "p95_ms": 266.398}, "triton_wait": {"avg_ms": 74.068, "count": 130, "max_ms": 95.233, "p95_ms": 92.469}, "visualize": {"avg_ms": 9.879, "count": 130, "max_ms": 12.215, "p95_ms": 10.971}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.783, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 230}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3960.375, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:31:04+09:00", "last_error_summary": "[h264 @ 0x555c5ad93c80] error while decoding MB 63 6, bytestream -11", "last_exit_at": "2026-07-01T16:31:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.7, "last_started_at": "2026-07-01T16:29:14+09:00", "pid": 8734, "process": {"exists": false, "pid": 8734}, "restart_count": 49, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.564, "sample_count": 1275, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27633, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:31:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T16:31:05+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T16:31:04+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T16:31:04.853792+09:00", "mtime_epoch": 1782891064.8537917, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200705}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:31:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 873.828, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.364, "pushed": 4.364, "read": 4.331}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.986, "count": 130, "max_ms": 9.467, "p95_ms": 8.759}, "gstreamer_push": {"avg_ms": 0.928, "count": 130, "max_ms": 2.976, "p95_ms": 1.701}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.459, "count": 130, "max_ms": 71.99, "p95_ms": 66.642}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.59, "count": 130, "max_ms": 90.168, "p95_ms": 86.184}, "yolo11_640_fire_11class": {"avg_ms": 58.345, "count": 130, "max_ms": 78.182, "p95_ms": 73.105}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.824, "count": 130, "max_ms": 95.221, "p95_ms": 92.456}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.248, "count": 130, "max_ms": 289.789, "p95_ms": 256.738}, "read_interval": {"avg_ms": 230.895, "count": 129, "max_ms": 299.974, "p95_ms": 266.398}, "triton_wait": {"avg_ms": 74.068, "count": 130, "max_ms": 95.233, "p95_ms": 92.469}, "visualize": {"avg_ms": 9.879, "count": 130, "max_ms": 12.215, "p95_ms": 10.971}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.786, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 230}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4065.879, "limit_mb": 8796093022207.996}, "detail": {"pid": 8841, "restart_count": 50, "start_count": 51}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:31:04+09:00", "last_error_summary": "[h264 @ 0x555c5ad93c80] error while decoding MB 63 6, bytestream -11", "last_exit_at": "2026-07-01T16:31:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:31:07+09:00", "pid": 8841, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.215, "name": "ffmpeg", "pid": 8841, "rss_mb": 139.215, "state": "S (sleeping)", "threads": 106}, "restart_count": 50, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.564, "sample_count": 1253, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 27633, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:31:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.927, "last_frame_pushed_at": "2026-07-01T16:31:05+09:00", "last_frame_read_age_sec": 5.152, "last_frame_read_at": "2026-07-01T16:31:04+09:00", "mjpeg": {"age_sec": 4.912, "exists": true, "mtime": "2026-07-01T16:31:05.085796+09:00", "mtime_epoch": 1782891065.0857956, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200802}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:31:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 876.938, "name": "celery", "pid": 237, "rss_mb": 873.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.646, "pushed": 3.646, "read": 3.613}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.044, "count": 108, "max_ms": 9.467, "p95_ms": 8.771}, "gstreamer_push": {"avg_ms": 0.914, "count": 109, "max_ms": 2.939, "p95_ms": 1.026}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.867, "count": 108, "max_ms": 71.99, "p95_ms": 65.725}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.317, "count": 108, "max_ms": 90.168, "p95_ms": 86.417}, "yolo11_640_fire_11class": {"avg_ms": 57.875, "count": 108, "max_ms": 75.408, "p95_ms": 73.092}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.506, "count": 108, "max_ms": 95.221, "p95_ms": 93.068}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.517, "count": 109, "max_ms": 289.789, "p95_ms": 256.738}, "read_interval": {"avg_ms": 231.119, "count": 108, "max_ms": 299.974, "p95_ms": 266.398}, "triton_wait": {"avg_ms": 73.796, "count": 108, "max_ms": 95.233, "p95_ms": 93.081}, "visualize": {"avg_ms": 9.92, "count": 108, "max_ms": 12.215, "p95_ms": 10.974}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 234}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3959.016, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55846304bfc0] error while decoding MB 64 17, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:32:58+09:00", "last_error_summary": "[h264 @ 0x55846304bfc0] error while decoding MB 64 17, bytestream -7", "last_exit_at": "2026-07-01T16:31:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.042, "last_started_at": "2026-07-01T16:31:07+09:00", "pid": 8841, "process": {"exists": false, "pid": 8841}, "restart_count": 50, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.793, "sample_count": 1256, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:32:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T16:32:57+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T16:32:57+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T16:32:57.999719+09:00", "mtime_epoch": 1782891177.9997191, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185484}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:32:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 877.246, "name": "celery", "pid": 237, "rss_mb": 871.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.061, "pushed": 4.061, "read": 4.061}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.497, "count": 121, "max_ms": 12.396, "p95_ms": 11.012}, "gstreamer_push": {"avg_ms": 0.915, "count": 121, "max_ms": 3.048, "p95_ms": 0.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.179, "count": 121, "max_ms": 70.626, "p95_ms": 62.53}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.653, "count": 121, "max_ms": 96.737, "p95_ms": 89.751}, "yolo11_640_fire_11class": {"avg_ms": 54.933, "count": 121, "max_ms": 76.758, "p95_ms": 71.397}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.434, "count": 121, "max_ms": 95.634, "p95_ms": 92.838}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.169, "count": 121, "max_ms": 333.52, "p95_ms": 276.224}, "read_interval": {"avg_ms": 247.58, "count": 121, "max_ms": 343.495, "p95_ms": 285.27}, "triton_wait": {"avg_ms": 69.702, "count": 121, "max_ms": 96.75, "p95_ms": 92.878}, "visualize": {"avg_ms": 11.039, "count": 121, "max_ms": 13.316, "p95_ms": 12.435}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.798, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 234}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3955.305, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:32:58+09:00", "last_error_summary": "[h264 @ 0x55846304bfc0] error while decoding MB 64 17, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:32:58+09:00", "last_error_summary": "[h264 @ 0x55846304bfc0] error while decoding MB 64 17, bytestream -7", "last_exit_at": "2026-07-01T16:32:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.265, "last_started_at": "2026-07-01T16:31:07+09:00", "pid": 8841, "process": {"exists": false, "pid": 8841}, "restart_count": 50, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.793, "sample_count": 1256, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28092, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:32:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:32:58+09:00", "last_frame_read_age_sec": 0.272, "last_frame_read_at": "2026-07-01T16:32:57+09:00", "mjpeg": {"age_sec": 0.262, "exists": true, "mtime": "2026-07-01T16:32:57.999719+09:00", "mtime_epoch": 1782891177.9997191, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185484}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:32:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 877.246, "name": "celery", "pid": 237, "rss_mb": 871.828, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.061, "pushed": 4.061, "read": 4.028}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.508, "count": 121, "max_ms": 12.396, "p95_ms": 11.012}, "gstreamer_push": {"avg_ms": 0.897, "count": 121, "max_ms": 2.83, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.363, "count": 121, "max_ms": 70.626, "p95_ms": 63.864}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.865, "count": 121, "max_ms": 96.737, "p95_ms": 89.751}, "yolo11_640_fire_11class": {"avg_ms": 55.069, "count": 121, "max_ms": 76.758, "p95_ms": 71.397}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.608, "count": 121, "max_ms": 95.634, "p95_ms": 92.838}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.654, "count": 121, "max_ms": 333.52, "p95_ms": 276.224}, "read_interval": {"avg_ms": 247.79, "count": 120, "max_ms": 343.495, "p95_ms": 285.27}, "triton_wait": {"avg_ms": 69.877, "count": 121, "max_ms": 96.75, "p95_ms": 92.878}, "visualize": {"avg_ms": 11.055, "count": 121, "max_ms": 13.316, "p95_ms": 12.435}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.793, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 234}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3955.297, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:32:58+09:00", "last_error_summary": "[h264 @ 0x55846304bfc0] error while decoding MB 64 17, bytestream -7", "last_exit_at": "2026-07-01T16:32:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.268, "last_started_at": "2026-07-01T16:31:07+09:00", "pid": 8841, "process": {"exists": false, "pid": 8841}, "restart_count": 50, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.793, "sample_count": 1256, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28092, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:32:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:32:58+09:00", "last_frame_read_age_sec": 0.276, "last_frame_read_at": "2026-07-01T16:32:57+09:00", "mjpeg": {"age_sec": 0.265, "exists": true, "mtime": "2026-07-01T16:32:57.999719+09:00", "mtime_epoch": 1782891177.9997191, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185484}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:32:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 877.246, "name": "celery", "pid": 237, "rss_mb": 871.828, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.061, "pushed": 4.061, "read": 4.027}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.508, "count": 121, "max_ms": 12.396, "p95_ms": 11.012}, "gstreamer_push": {"avg_ms": 0.897, "count": 121, "max_ms": 2.83, "p95_ms": 0.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.363, "count": 121, "max_ms": 70.626, "p95_ms": 63.864}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.865, "count": 121, "max_ms": 96.737, "p95_ms": 89.751}, "yolo11_640_fire_11class": {"avg_ms": 55.069, "count": 121, "max_ms": 76.758, "p95_ms": 71.397}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.608, "count": 121, "max_ms": 95.634, "p95_ms": 92.838}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.654, "count": 121, "max_ms": 333.52, "p95_ms": 276.224}, "read_interval": {"avg_ms": 247.79, "count": 120, "max_ms": 343.495, "p95_ms": 285.27}, "triton_wait": {"avg_ms": 69.877, "count": 121, "max_ms": 96.75, "p95_ms": 92.878}, "visualize": {"avg_ms": 11.055, "count": 121, "max_ms": 13.316, "p95_ms": 12.435}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.796, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 234}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4065.699, "limit_mb": 8796093022207.996}, "detail": {"pid": 8948, "restart_count": 51, "start_count": 52}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:32:58+09:00", "last_error_summary": "[h264 @ 0x55846304bfc0] error while decoding MB 64 17, bytestream -7", "last_exit_at": "2026-07-01T16:33:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:33:00+09:00", "pid": 8948, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.664, "name": "ffmpeg", "pid": 8948, "rss_mb": 136.664, "state": "S (sleeping)", "threads": 106}, "restart_count": 51, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.949, "sample_count": 1235, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28092, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:32:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.576, "last_frame_pushed_at": "2026-07-01T16:32:58+09:00", "last_frame_read_age_sec": 4.846, "last_frame_read_at": "2026-07-01T16:32:57+09:00", "mjpeg": {"age_sec": 4.559, "exists": true, "mtime": "2026-07-01T16:32:58.275724+09:00", "mtime_epoch": 1782891178.2757237, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184218}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:33:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 877.246, "name": "celery", "pid": 237, "rss_mb": 871.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.388, "pushed": 3.388, "read": 3.355}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.703, "count": 101, "max_ms": 12.396, "p95_ms": 11.017}, "gstreamer_push": {"avg_ms": 0.885, "count": 101, "max_ms": 1.795, "p95_ms": 0.906}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.936, "count": 101, "max_ms": 70.626, "p95_ms": 63.864}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.839, "count": 101, "max_ms": 96.737, "p95_ms": 89.751}, "yolo11_640_fire_11class": {"avg_ms": 55.587, "count": 101, "max_ms": 76.758, "p95_ms": 71.345}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.357, "count": 101, "max_ms": 95.634, "p95_ms": 92.829}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.873, "count": 101, "max_ms": 333.52, "p95_ms": 277.528}, "read_interval": {"avg_ms": 251.938, "count": 100, "max_ms": 343.495, "p95_ms": 286.491}, "triton_wait": {"avg_ms": 70.676, "count": 101, "max_ms": 96.75, "p95_ms": 92.851}, "visualize": {"avg_ms": 11.211, "count": 101, "max_ms": 13.316, "p95_ms": 12.523}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.809, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3960.875, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "last_exit_at": "2026-07-01T16:33:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.497, "last_started_at": "2026-07-01T16:33:00+09:00", "pid": 8948, "process": {"exists": false, "pid": 8948}, "restart_count": 51, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.9, "sample_count": 1233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28536, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:34:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-01T16:34:49+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-01T16:34:49+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-01T16:34:49.281615+09:00", "mtime_epoch": 1782891289.2816153, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:34:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 877.246, "name": "celery", "pid": 237, "rss_mb": 873.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.368, "pushed": 4.368, "read": 4.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.636, "count": 130, "max_ms": 12.943, "p95_ms": 10.006}, "gstreamer_push": {"avg_ms": 0.97, "count": 130, "max_ms": 2.801, "p95_ms": 1.806}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.184, "count": 130, "max_ms": 76.186, "p95_ms": 61.762}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.931, "count": 130, "max_ms": 97.77, "p95_ms": 93.235}, "yolo11_640_fire_11class": {"avg_ms": 52.899, "count": 130, "max_ms": 75.949, "p95_ms": 72.48}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.953, "count": 130, "max_ms": 96.681, "p95_ms": 93.068}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.648, "count": 130, "max_ms": 316.556, "p95_ms": 259.052}, "read_interval": {"avg_ms": 230.346, "count": 130, "max_ms": 326.492, "p95_ms": 268.802}, "triton_wait": {"avg_ms": 66.659, "count": 130, "max_ms": 97.783, "p95_ms": 94.004}, "visualize": {"avg_ms": 10.298, "count": 130, "max_ms": 12.104, "p95_ms": 11.659}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.765, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3959.75, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "last_exit_at": "2026-07-01T16:34:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.689, "last_started_at": "2026-07-01T16:33:00+09:00", "pid": 8948, "process": {"exists": false, "pid": 8948}, "restart_count": 51, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.9, "sample_count": 1233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28537, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:34:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:34:49+09:00", "last_frame_read_age_sec": 0.249, "last_frame_read_at": "2026-07-01T16:34:49+09:00", "mjpeg": {"age_sec": 0.241, "exists": true, "mtime": "2026-07-01T16:34:49.281615+09:00", "mtime_epoch": 1782891289.2816153, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:34:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 877.246, "name": "celery", "pid": 237, "rss_mb": 873.984, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.373, "pushed": 4.373, "read": 4.34}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.641, "count": 130, "max_ms": 12.943, "p95_ms": 10.006}, "gstreamer_push": {"avg_ms": 0.969, "count": 131, "max_ms": 2.801, "p95_ms": 1.806}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.28, "count": 130, "max_ms": 76.186, "p95_ms": 62.519}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.85, "count": 130, "max_ms": 97.77, "p95_ms": 93.235}, "yolo11_640_fire_11class": {"avg_ms": 52.872, "count": 130, "max_ms": 75.949, "p95_ms": 72.48}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.886, "count": 130, "max_ms": 96.681, "p95_ms": 92.512}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.833, "count": 131, "max_ms": 316.556, "p95_ms": 259.052}, "read_interval": {"avg_ms": 230.346, "count": 130, "max_ms": 326.492, "p95_ms": 268.802}, "triton_wait": {"avg_ms": 66.592, "count": 130, "max_ms": 97.783, "p95_ms": 94.004}, "visualize": {"avg_ms": 10.302, "count": 130, "max_ms": 12.104, "p95_ms": 11.659}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3959.719, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "last_exit_at": "2026-07-01T16:34:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.692, "last_started_at": "2026-07-01T16:33:00+09:00", "pid": 8948, "process": {"exists": false, "pid": 8948}, "restart_count": 51, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.9, "sample_count": 1233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28537, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:34:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:34:49+09:00", "last_frame_read_age_sec": 0.252, "last_frame_read_at": "2026-07-01T16:34:49+09:00", "mjpeg": {"age_sec": 0.244, "exists": true, "mtime": "2026-07-01T16:34:49.281615+09:00", "mtime_epoch": 1782891289.2816153, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:34:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 877.246, "name": "celery", "pid": 237, "rss_mb": 873.984, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.372, "pushed": 4.372, "read": 4.339}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.641, "count": 130, "max_ms": 12.943, "p95_ms": 10.006}, "gstreamer_push": {"avg_ms": 0.969, "count": 131, "max_ms": 2.801, "p95_ms": 1.806}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.28, "count": 130, "max_ms": 76.186, "p95_ms": 62.519}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.85, "count": 130, "max_ms": 97.77, "p95_ms": 93.235}, "yolo11_640_fire_11class": {"avg_ms": 52.872, "count": 130, "max_ms": 75.949, "p95_ms": 72.48}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.886, "count": 130, "max_ms": 96.681, "p95_ms": 92.512}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.833, "count": 131, "max_ms": 316.556, "p95_ms": 259.052}, "read_interval": {"avg_ms": 230.346, "count": 130, "max_ms": 326.492, "p95_ms": 268.802}, "triton_wait": {"avg_ms": 66.592, "count": 130, "max_ms": 97.783, "p95_ms": 94.004}, "visualize": {"avg_ms": 10.302, "count": 130, "max_ms": 12.104, "p95_ms": 11.659}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 237}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4070.367, "limit_mb": 8796093022207.996}, "detail": {"pid": 9055, "restart_count": 52, "start_count": 53}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "last_exit_at": "2026-07-01T16:34:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:34:52+09:00", "pid": 9055, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.883, "name": "ffmpeg", "pid": 9055, "rss_mb": 138.883, "state": "S (sleeping)", "threads": 106}, "restart_count": 52, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 260.254, "sample_count": 1211, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 28537, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:34:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.974, "last_frame_pushed_at": "2026-07-01T16:34:49+09:00", "last_frame_read_age_sec": 5.221, "last_frame_read_at": "2026-07-01T16:34:49+09:00", "mjpeg": {"age_sec": 4.957, "exists": true, "mtime": "2026-07-01T16:34:49.537620+09:00", "mtime_epoch": 1782891289.5376196, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186486}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:34:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 877.246, "name": "celery", "pid": 237, "rss_mb": 873.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.622, "pushed": 3.622, "read": 3.589}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.601, "count": 108, "max_ms": 12.943, "p95_ms": 9.782}, "gstreamer_push": {"avg_ms": 0.964, "count": 108, "max_ms": 2.801, "p95_ms": 1.801}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.751, "count": 108, "max_ms": 76.186, "p95_ms": 63.665}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.635, "count": 108, "max_ms": 97.77, "p95_ms": 93.667}, "yolo11_640_fire_11class": {"avg_ms": 53.404, "count": 108, "max_ms": 75.949, "p95_ms": 73.889}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.575, "count": 108, "max_ms": 96.681, "p95_ms": 93.068}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.805, "count": 108, "max_ms": 316.556, "p95_ms": 259.65}, "read_interval": {"avg_ms": 232.296, "count": 107, "max_ms": 326.492, "p95_ms": 269.24}, "triton_wait": {"avg_ms": 67.412, "count": 108, "max_ms": 97.783, "p95_ms": 94.102}, "visualize": {"avg_ms": 10.345, "count": 108, "max_ms": 11.972, "p95_ms": 11.659}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.817, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3893.992, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ec8e23b000] error while decoding MB 21 46, bytestream -7 [in#0/rtsp @ 0x55ec8e1f3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ec8e1f3e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:35:30+09:00", "last_error_summary": "[h264 @ 0x55ec8e23b000] error while decoding MB 21 46, bytestream -7 [in#0/rtsp @ 0x55ec8e1f3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ec8e1f3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T16:21:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 827.629, "last_started_at": "2026-07-01T16:21:42+09:00", "pid": 8261, "process": {"exists": false, "pid": 8261}, "restart_count": 6, "restart_count_per_hour": 3, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.95, "sample_count": 2568, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58843, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:35:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T16:35:30+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-01T16:35:30+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-01T16:35:30.466317+09:00", "mtime_epoch": 1782891330.4663172, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:35:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 892.535, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 21, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.701, "pushed": 0.701, "read": 0.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.67, "count": 20, "max_ms": 12.476, "p95_ms": 12.251}, "gstreamer_push": {"avg_ms": 0.954, "count": 21, "max_ms": 2.182, "p95_ms": 1.128}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.132, "count": 20, "max_ms": 71.87, "p95_ms": 68.551}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.302, "count": 20, "max_ms": 97.909, "p95_ms": 96.144}, "yolo11_640_fire_11class": {"avg_ms": 56.359, "count": 20, "max_ms": 76.949, "p95_ms": 71.883}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.826, "count": 20, "max_ms": 98.249, "p95_ms": 89.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.4, "count": 21, "max_ms": 250.411, "p95_ms": 227.017}, "read_interval": {"avg_ms": 1431.087, "count": 21, "max_ms": 15980.952, "p95_ms": 10240.109}, "triton_wait": {"avg_ms": 66.575, "count": 20, "max_ms": 98.262, "p95_ms": 97.923}, "visualize": {"avg_ms": 6.347, "count": 20, "max_ms": 7.443, "p95_ms": 7.3}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3892.777, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:35:30+09:00", "last_error_summary": "[h264 @ 0x55ec8e23b000] error while decoding MB 21 46, bytestream -7 [in#0/rtsp @ 0x55ec8e1f3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ec8e1f3e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:35:30+09:00", "last_error_summary": "[h264 @ 0x55ec8e23b000] error while decoding MB 21 46, bytestream -7 [in#0/rtsp @ 0x55ec8e1f3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ec8e1f3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T16:35:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 827.697, "last_started_at": "2026-07-01T16:21:42+09:00", "pid": 8261, "process": {"exists": false, "pid": 8261}, "restart_count": 6, "restart_count_per_hour": 3, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.95, "sample_count": 2569, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58844, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:35:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T16:35:30+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-01T16:35:30+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-01T16:35:30.466317+09:00", "mtime_epoch": 1782891330.4663172, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:35:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 892.535, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 21, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.7, "pushed": 0.7, "read": 0.7}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.748, "count": 21, "max_ms": 12.476, "p95_ms": 12.308}, "gstreamer_push": {"avg_ms": 0.965, "count": 21, "max_ms": 2.182, "p95_ms": 1.133}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.762, "count": 21, "max_ms": 71.87, "p95_ms": 68.551}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.136, "count": 21, "max_ms": 97.909, "p95_ms": 96.144}, "yolo11_640_fire_11class": {"avg_ms": 56.296, "count": 21, "max_ms": 76.949, "p95_ms": 71.883}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.441, "count": 21, "max_ms": 98.249, "p95_ms": 89.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 189.01, "count": 21, "max_ms": 250.411, "p95_ms": 227.017}, "read_interval": {"avg_ms": 1431.087, "count": 21, "max_ms": 15980.952, "p95_ms": 10240.109}, "triton_wait": {"avg_ms": 66.301, "count": 21, "max_ms": 98.262, "p95_ms": 97.923}, "visualize": {"avg_ms": 6.373, "count": 21, "max_ms": 7.443, "p95_ms": 7.3}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3892.77, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:35:30+09:00", "last_error_summary": "[h264 @ 0x55ec8e23b000] error while decoding MB 21 46, bytestream -7 [in#0/rtsp @ 0x55ec8e1f3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ec8e1f3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T16:35:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 827.702, "last_started_at": "2026-07-01T16:21:42+09:00", "pid": 8261, "process": {"exists": false, "pid": 8261}, "restart_count": 6, "restart_count_per_hour": 3, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.95, "sample_count": 2569, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58844, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:35:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T16:35:30+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-01T16:35:30+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-01T16:35:30.466317+09:00", "mtime_epoch": 1782891330.4663172, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:35:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 892.535, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 20, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.704, "pushed": 0.704, "read": 0.671}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.748, "count": 21, "max_ms": 12.476, "p95_ms": 12.308}, "gstreamer_push": {"avg_ms": 0.965, "count": 21, "max_ms": 2.182, "p95_ms": 1.133}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.762, "count": 21, "max_ms": 71.87, "p95_ms": 68.551}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.136, "count": 21, "max_ms": 97.909, "p95_ms": 96.144}, "yolo11_640_fire_11class": {"avg_ms": 56.296, "count": 21, "max_ms": 76.949, "p95_ms": 71.883}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.441, "count": 21, "max_ms": 98.249, "p95_ms": 89.404}}, "truncated_count": 0}, "processing_total": {"avg_ms": 189.01, "count": 21, "max_ms": 250.411, "p95_ms": 227.017}, "read_interval": {"avg_ms": 1493.362, "count": 20, "max_ms": 15980.952, "p95_ms": 10240.109}, "triton_wait": {"avg_ms": 66.301, "count": 21, "max_ms": 98.262, "p95_ms": 97.923}, "visualize": {"avg_ms": 6.373, "count": 21, "max_ms": 7.443, "p95_ms": 7.3}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.811, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 490}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4014.523, "limit_mb": 8796093022207.996}, "detail": {"pid": 9162, "restart_count": 7, "start_count": 8}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:35:30+09:00", "last_error_summary": "[h264 @ 0x55ec8e23b000] error while decoding MB 21 46, bytestream -7 [in#0/rtsp @ 0x55ec8e1f3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ec8e1f3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T16:35:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:35:32+09:00", "pid": 9162, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.555, "name": "ffmpeg", "pid": 9162, "rss_mb": 156.555, "state": "R (running)", "threads": 49}, "restart_count": 7, "restart_count_per_hour": 4, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.039, "sample_count": 2536, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58844, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:35:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.055, "last_frame_pushed_at": "2026-07-01T16:35:30+09:00", "last_frame_read_age_sec": 4.182, "last_frame_read_at": "2026-07-01T16:35:30+09:00", "mjpeg": {"age_sec": 4.036, "exists": true, "mtime": "2026-07-01T16:35:30.606320+09:00", "mtime_epoch": 1782891330.6063197, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 252072}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T16:35:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1147.648, "name": "celery", "pid": 267, "rss_mb": 892.535, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.619, "pushed": 0.619, "read": 0.619}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.762, "count": 14, "max_ms": 12.476, "p95_ms": 12.476}, "gstreamer_push": {"avg_ms": 1.007, "count": 14, "max_ms": 2.182, "p95_ms": 2.182}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.232, "count": 14, "max_ms": 71.87, "p95_ms": 71.87}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.409, "count": 14, "max_ms": 97.909, "p95_ms": 97.909}, "yolo11_640_fire_11class": {"avg_ms": 58.584, "count": 14, "max_ms": 76.949, "p95_ms": 76.949}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.744, "count": 14, "max_ms": 98.249, "p95_ms": 98.249}}, "truncated_count": 0}, "processing_total": {"avg_ms": 188.221, "count": 14, "max_ms": 250.411, "p95_ms": 250.411}, "read_interval": {"avg_ms": 2047.295, "count": 14, "max_ms": 15980.952, "p95_ms": 15980.952}, "triton_wait": {"avg_ms": 69.529, "count": 14, "max_ms": 98.262, "p95_ms": 98.262}, "visualize": {"avg_ms": 6.516, "count": 14, "max_ms": 7.443, "p95_ms": 7.443}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.604, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 241}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3956.539, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "last_exit_at": "2026-07-01T16:36:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.521, "last_started_at": "2026-07-01T16:34:52+09:00", "pid": 9055, "process": {"exists": false, "pid": 9055}, "restart_count": 52, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.459, "sample_count": 1233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29002, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:36:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:36:41+09:00", "last_frame_read_age_sec": 0.242, "last_frame_read_at": "2026-07-01T16:36:40+09:00", "mjpeg": {"age_sec": 0.23, "exists": true, "mtime": "2026-07-01T16:36:40.783516+09:00", "mtime_epoch": 1782891400.7835157, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177944}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:36:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 877.199, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.427, "pushed": 4.427, "read": 4.393}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.605, "count": 132, "max_ms": 14.056, "p95_ms": 10.442}, "gstreamer_push": {"avg_ms": 0.967, "count": 132, "max_ms": 2.894, "p95_ms": 1.79}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.519, "count": 132, "max_ms": 71.868, "p95_ms": 64.852}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.629, "count": 132, "max_ms": 95.645, "p95_ms": 91.62}, "yolo11_640_fire_11class": {"avg_ms": 53.442, "count": 132, "max_ms": 73.628, "p95_ms": 70.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.238, "count": 132, "max_ms": 98.247, "p95_ms": 92.117}}, "truncated_count": 0}, "processing_total": {"avg_ms": 217.906, "count": 132, "max_ms": 285.909, "p95_ms": 255.598}, "read_interval": {"avg_ms": 227.341, "count": 131, "max_ms": 297.185, "p95_ms": 264.854}, "triton_wait": {"avg_ms": 68.016, "count": 132, "max_ms": 98.259, "p95_ms": 92.332}, "visualize": {"avg_ms": 10.1, "count": 132, "max_ms": 14.556, "p95_ms": 11.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.818, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 241}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3956.656, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "last_exit_at": "2026-07-01T16:36:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.524, "last_started_at": "2026-07-01T16:34:52+09:00", "pid": 9055, "process": {"exists": false, "pid": 9055}, "restart_count": 52, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.459, "sample_count": 1233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29002, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:36:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:36:41+09:00", "last_frame_read_age_sec": 0.245, "last_frame_read_at": "2026-07-01T16:36:40+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-01T16:36:40.783516+09:00", "mtime_epoch": 1782891400.7835157, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177944}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:36:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 877.199, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.426, "pushed": 4.426, "read": 4.393}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.605, "count": 132, "max_ms": 14.056, "p95_ms": 10.442}, "gstreamer_push": {"avg_ms": 0.967, "count": 132, "max_ms": 2.894, "p95_ms": 1.79}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.519, "count": 132, "max_ms": 71.868, "p95_ms": 64.852}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.629, "count": 132, "max_ms": 95.645, "p95_ms": 91.62}, "yolo11_640_fire_11class": {"avg_ms": 53.442, "count": 132, "max_ms": 73.628, "p95_ms": 70.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.238, "count": 132, "max_ms": 98.247, "p95_ms": 92.117}}, "truncated_count": 0}, "processing_total": {"avg_ms": 217.906, "count": 132, "max_ms": 285.909, "p95_ms": 255.598}, "read_interval": {"avg_ms": 227.341, "count": 131, "max_ms": 297.185, "p95_ms": 264.854}, "triton_wait": {"avg_ms": 68.016, "count": 132, "max_ms": 98.259, "p95_ms": 92.332}, "visualize": {"avg_ms": 10.1, "count": 132, "max_ms": 14.556, "p95_ms": 11.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.822, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 241}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4067.336, "limit_mb": 8796093022207.996}, "detail": {"pid": 9314, "restart_count": 53, "start_count": 54}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "last_exit_at": "2026-07-01T16:36:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:36:43+09:00", "pid": 9314, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.473, "name": "ffmpeg", "pid": 9314, "rss_mb": 138.473, "state": "S (sleeping)", "threads": 106}, "restart_count": 53, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.459, "sample_count": 1212, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29002, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:36:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.857, "last_frame_pushed_at": "2026-07-01T16:36:41+09:00", "last_frame_read_age_sec": 5.096, "last_frame_read_at": "2026-07-01T16:36:40+09:00", "mjpeg": {"age_sec": 4.84, "exists": true, "mtime": "2026-07-01T16:36:41.027520+09:00", "mtime_epoch": 1782891401.02752, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176706}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:36:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 877.199, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.713, "pushed": 3.713, "read": 3.68}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.617, "count": 110, "max_ms": 14.056, "p95_ms": 10.442}, "gstreamer_push": {"avg_ms": 0.937, "count": 111, "max_ms": 1.931, "p95_ms": 1.763}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.438, "count": 110, "max_ms": 69.108, "p95_ms": 64.74}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.526, "count": 110, "max_ms": 95.645, "p95_ms": 91.537}, "yolo11_640_fire_11class": {"avg_ms": 53.562, "count": 110, "max_ms": 73.628, "p95_ms": 70.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.271, "count": 110, "max_ms": 98.247, "p95_ms": 91.53}}, "truncated_count": 0}, "processing_total": {"avg_ms": 217.962, "count": 111, "max_ms": 285.909, "p95_ms": 251.512}, "read_interval": {"avg_ms": 227.372, "count": 110, "max_ms": 297.185, "p95_ms": 261.64}, "triton_wait": {"avg_ms": 67.855, "count": 110, "max_ms": 98.259, "p95_ms": 92.142}, "visualize": {"avg_ms": 10.133, "count": 110, "max_ms": 14.556, "p95_ms": 11.82}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 245}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3950.703, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "last_exit_at": "2026-07-01T16:38:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 117.098, "last_started_at": "2026-07-01T16:36:43+09:00", "pid": 9314, "process": {"exists": false, "pid": 9314}, "restart_count": 53, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.601, "sample_count": 1210, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:38:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:38:40+09:00", "last_frame_read_age_sec": 0.283, "last_frame_read_at": "2026-07-01T16:38:40+09:00", "mjpeg": {"age_sec": 0.269, "exists": true, "mtime": "2026-07-01T16:38:40.694833+09:00", "mtime_epoch": 1782891520.6948333, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202870}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:38:40+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 866.387, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.017, "pushed": 4.017, "read": 3.983}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.691, "count": 119, "max_ms": 12.143, "p95_ms": 11.526}, "gstreamer_push": {"avg_ms": 0.949, "count": 120, "max_ms": 2.9, "p95_ms": 1.273}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.569, "count": 119, "max_ms": 70.253, "p95_ms": 61.262}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.289, "count": 119, "max_ms": 90.278, "p95_ms": 85.789}, "yolo11_640_fire_11class": {"avg_ms": 52.878, "count": 119, "max_ms": 74.131, "p95_ms": 70.682}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.381, "count": 119, "max_ms": 94.438, "p95_ms": 91.002}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.861, "count": 120, "max_ms": 348.394, "p95_ms": 276.128}, "read_interval": {"avg_ms": 250.793, "count": 119, "max_ms": 359.489, "p95_ms": 286.372}, "triton_wait": {"avg_ms": 66.432, "count": 119, "max_ms": 94.451, "p95_ms": 91.014}, "visualize": {"avg_ms": 11.366, "count": 119, "max_ms": 14.32, "p95_ms": 12.595}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 245}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3950.695, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "last_exit_at": "2026-07-01T16:38:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 117.101, "last_started_at": "2026-07-01T16:36:43+09:00", "pid": 9314, "process": {"exists": false, "pid": 9314}, "restart_count": 53, "restart_count_per_hour": 28, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.601, "sample_count": 1210, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:38:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:38:40+09:00", "last_frame_read_age_sec": 0.287, "last_frame_read_at": "2026-07-01T16:38:40+09:00", "mjpeg": {"age_sec": 0.273, "exists": true, "mtime": "2026-07-01T16:38:40.694833+09:00", "mtime_epoch": 1782891520.6948333, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202870}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:38:40+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 866.387, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.016, "pushed": 4.016, "read": 3.983}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.691, "count": 119, "max_ms": 12.143, "p95_ms": 11.526}, "gstreamer_push": {"avg_ms": 0.949, "count": 120, "max_ms": 2.9, "p95_ms": 1.273}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.569, "count": 119, "max_ms": 70.253, "p95_ms": 61.262}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.289, "count": 119, "max_ms": 90.278, "p95_ms": 85.789}, "yolo11_640_fire_11class": {"avg_ms": 52.878, "count": 119, "max_ms": 74.131, "p95_ms": 70.682}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.381, "count": 119, "max_ms": 94.438, "p95_ms": 91.002}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.861, "count": 120, "max_ms": 348.394, "p95_ms": 276.128}, "read_interval": {"avg_ms": 250.793, "count": 119, "max_ms": 359.489, "p95_ms": 286.372}, "triton_wait": {"avg_ms": 66.432, "count": 119, "max_ms": 94.451, "p95_ms": 91.014}, "visualize": {"avg_ms": 11.366, "count": 119, "max_ms": 14.32, "p95_ms": 12.595}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 245}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4061.051, "limit_mb": 8796093022207.996}, "detail": {"pid": 9421, "restart_count": 54, "start_count": 55}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:34:49+09:00", "last_error_summary": "[h264 @ 0x56262a4a32c0] error while decoding MB 9 24, bytestream -13", "last_exit_at": "2026-07-01T16:38:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:38:43+09:00", "pid": 9421, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.527, "name": "ffmpeg", "pid": 9421, "rss_mb": 141.527, "state": "S (sleeping)", "threads": 106}, "restart_count": 54, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.411, "sample_count": 1192, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:38:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.553, "last_frame_pushed_at": "2026-07-01T16:38:40+09:00", "last_frame_read_age_sec": 4.833, "last_frame_read_at": "2026-07-01T16:38:40+09:00", "mjpeg": {"age_sec": 4.535, "exists": true, "mtime": "2026-07-01T16:38:40.978843+09:00", "mtime_epoch": 1782891520.9788432, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203059}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:38:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 866.387, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.434, "pushed": 3.434, "read": 3.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.627, "count": 102, "max_ms": 12.143, "p95_ms": 11.526}, "gstreamer_push": {"avg_ms": 0.924, "count": 103, "max_ms": 2.9, "p95_ms": 1.019}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.008, "count": 102, "max_ms": 70.253, "p95_ms": 61.262}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.007, "count": 102, "max_ms": 90.278, "p95_ms": 85.789}, "yolo11_640_fire_11class": {"avg_ms": 53.426, "count": 102, "max_ms": 74.131, "p95_ms": 70.682}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.108, "count": 102, "max_ms": 94.438, "p95_ms": 91.002}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.365, "count": 103, "max_ms": 348.394, "p95_ms": 276.128}, "read_interval": {"avg_ms": 249.093, "count": 102, "max_ms": 359.489, "p95_ms": 285.283}, "triton_wait": {"avg_ms": 67.164, "count": 102, "max_ms": 94.451, "p95_ms": 91.014}, "visualize": {"avg_ms": 11.298, "count": 102, "max_ms": 14.32, "p95_ms": 12.595}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 249}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3964.797, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559058d65b00] error while decoding MB 62 41, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:40:33+09:00", "last_error_summary": "[h264 @ 0x559058d65b00] error while decoding MB 62 41, bytestream -5", "last_exit_at": "2026-07-01T16:38:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.638, "last_started_at": "2026-07-01T16:38:43+09:00", "pid": 9421, "process": {"exists": false, "pid": 9421}, "restart_count": 54, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.026, "sample_count": 1191, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29902, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:40:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.053, "last_frame_pushed_at": "2026-07-01T16:40:33+09:00", "last_frame_read_age_sec": 0.045, "last_frame_read_at": "2026-07-01T16:40:33+09:00", "mjpeg": {"age_sec": 0.036, "exists": true, "mtime": "2026-07-01T16:40:33.114622+09:00", "mtime_epoch": 1782891633.114622, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194919}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:40:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 877.91, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.151, "pushed": 4.151, "read": 4.151}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.218, "count": 123, "max_ms": 12.048, "p95_ms": 10.973}, "gstreamer_push": {"avg_ms": 0.911, "count": 124, "max_ms": 2.951, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.882, "count": 123, "max_ms": 69.157, "p95_ms": 62.607}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.303, "count": 123, "max_ms": 93.427, "p95_ms": 89.408}, "yolo11_640_fire_11class": {"avg_ms": 55.521, "count": 123, "max_ms": 75.112, "p95_ms": 72.844}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.432, "count": 123, "max_ms": 97.086, "p95_ms": 92.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.914, "count": 124, "max_ms": 369.031, "p95_ms": 271.026}, "read_interval": {"avg_ms": 242.484, "count": 124, "max_ms": 379.205, "p95_ms": 280.735}, "triton_wait": {"avg_ms": 70.013, "count": 123, "max_ms": 97.099, "p95_ms": 92.255}, "visualize": {"avg_ms": 10.615, "count": 123, "max_ms": 12.703, "p95_ms": 11.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 249}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3963.715, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:40:33+09:00", "last_error_summary": "[h264 @ 0x559058d65b00] error while decoding MB 62 41, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:40:33+09:00", "last_error_summary": "[h264 @ 0x559058d65b00] error while decoding MB 62 41, bytestream -5", "last_exit_at": "2026-07-01T16:40:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.825, "last_started_at": "2026-07-01T16:38:43+09:00", "pid": 9421, "process": {"exists": false, "pid": 9421}, "restart_count": 54, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.026, "sample_count": 1191, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:40:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:40:33+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T16:40:33+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T16:40:33.114622+09:00", "mtime_epoch": 1782891633.114622, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194919}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:40:33+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 877.91, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.161, "pushed": 4.161, "read": 4.128}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.217, "count": 124, "max_ms": 12.048, "p95_ms": 10.973}, "gstreamer_push": {"avg_ms": 0.911, "count": 124, "max_ms": 2.951, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.959, "count": 124, "max_ms": 69.157, "p95_ms": 62.607}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.288, "count": 124, "max_ms": 93.427, "p95_ms": 89.408}, "yolo11_640_fire_11class": {"avg_ms": 55.53, "count": 124, "max_ms": 75.112, "p95_ms": 72.844}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.414, "count": 124, "max_ms": 97.086, "p95_ms": 92.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.801, "count": 124, "max_ms": 369.031, "p95_ms": 271.026}, "read_interval": {"avg_ms": 242.411, "count": 123, "max_ms": 379.205, "p95_ms": 280.735}, "triton_wait": {"avg_ms": 69.991, "count": 124, "max_ms": 97.099, "p95_ms": 92.255}, "visualize": {"avg_ms": 10.614, "count": 124, "max_ms": 12.703, "p95_ms": 11.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.799, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 249}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3963.648, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:40:33+09:00", "last_error_summary": "[h264 @ 0x559058d65b00] error while decoding MB 62 41, bytestream -5", "last_exit_at": "2026-07-01T16:40:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.828, "last_started_at": "2026-07-01T16:38:43+09:00", "pid": 9421, "process": {"exists": false, "pid": 9421}, "restart_count": 54, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.026, "sample_count": 1191, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:40:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:40:33+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-01T16:40:33+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-01T16:40:33.114622+09:00", "mtime_epoch": 1782891633.114622, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194919}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:40:33+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 877.91, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.161, "pushed": 4.161, "read": 4.127}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.217, "count": 124, "max_ms": 12.048, "p95_ms": 10.973}, "gstreamer_push": {"avg_ms": 0.911, "count": 124, "max_ms": 2.951, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.959, "count": 124, "max_ms": 69.157, "p95_ms": 62.607}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.288, "count": 124, "max_ms": 93.427, "p95_ms": 89.408}, "yolo11_640_fire_11class": {"avg_ms": 55.53, "count": 124, "max_ms": 75.112, "p95_ms": 72.844}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.414, "count": 124, "max_ms": 97.086, "p95_ms": 92.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.801, "count": 124, "max_ms": 369.031, "p95_ms": 271.026}, "read_interval": {"avg_ms": 242.411, "count": 123, "max_ms": 379.205, "p95_ms": 280.735}, "triton_wait": {"avg_ms": 69.991, "count": 124, "max_ms": 97.099, "p95_ms": 92.255}, "visualize": {"avg_ms": 10.614, "count": 124, "max_ms": 12.703, "p95_ms": 11.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.802, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 249}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4075.215, "limit_mb": 8796093022207.996}, "detail": {"pid": 9528, "restart_count": 55, "start_count": 56}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:40:33+09:00", "last_error_summary": "[h264 @ 0x559058d65b00] error while decoding MB 62 41, bytestream -5", "last_exit_at": "2026-07-01T16:40:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:40:35+09:00", "pid": 9528, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.137, "name": "ffmpeg", "pid": 9528, "rss_mb": 142.137, "state": "S (sleeping)", "threads": 106}, "restart_count": 55, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.026, "sample_count": 1174, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 29903, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:40:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.118, "last_frame_pushed_at": "2026-07-01T16:40:33+09:00", "last_frame_read_age_sec": 4.347, "last_frame_read_at": "2026-07-01T16:40:33+09:00", "mjpeg": {"age_sec": 4.102, "exists": true, "mtime": "2026-07-01T16:40:33.350630+09:00", "mtime_epoch": 1782891633.3506298, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194834}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:40:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 877.91, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.567, "pushed": 3.567, "read": 3.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.245, "count": 107, "max_ms": 12.048, "p95_ms": 10.973}, "gstreamer_push": {"avg_ms": 0.92, "count": 107, "max_ms": 2.951, "p95_ms": 0.906}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.949, "count": 107, "max_ms": 69.157, "p95_ms": 62.607}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.208, "count": 107, "max_ms": 93.427, "p95_ms": 89.492}, "yolo11_640_fire_11class": {"avg_ms": 55.479, "count": 107, "max_ms": 75.112, "p95_ms": 72.844}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.331, "count": 107, "max_ms": 97.086, "p95_ms": 92.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.346, "count": 107, "max_ms": 369.031, "p95_ms": 271.026}, "read_interval": {"avg_ms": 241.885, "count": 107, "max_ms": 379.205, "p95_ms": 280.735}, "triton_wait": {"avg_ms": 69.93, "count": 107, "max_ms": 97.099, "p95_ms": 92.255}, "visualize": {"avg_ms": 10.597, "count": 107, "max_ms": 12.703, "p95_ms": 11.935}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 253}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3963.332, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56057acc4d80] error while decoding MB 8 18, bytestream -26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:42:26+09:00", "last_error_summary": "[h264 @ 0x56057acc4d80] error while decoding MB 8 18, bytestream -26", "last_exit_at": "2026-07-01T16:40:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.291, "last_started_at": "2026-07-01T16:40:35+09:00", "pid": 9528, "process": {"exists": false, "pid": 9528}, "restart_count": 55, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.345, "sample_count": 1201, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30362, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:42:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T16:42:26+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T16:42:26+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T16:42:26.698127+09:00", "mtime_epoch": 1782891746.6981268, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192585}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:42:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 875.602, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.182, "pushed": 4.182, "read": 4.182}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.928, "count": 124, "max_ms": 10.997, "p95_ms": 9.514}, "gstreamer_push": {"avg_ms": 0.897, "count": 125, "max_ms": 2.811, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.624, "count": 124, "max_ms": 69.637, "p95_ms": 62.762}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.924, "count": 124, "max_ms": 97.099, "p95_ms": 88.454}, "yolo11_640_fire_11class": {"avg_ms": 55.428, "count": 124, "max_ms": 74.843, "p95_ms": 73.317}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.04, "count": 124, "max_ms": 97.908, "p95_ms": 91.207}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.425, "count": 125, "max_ms": 284.867, "p95_ms": 255.376}, "read_interval": {"avg_ms": 240.72, "count": 125, "max_ms": 294.952, "p95_ms": 266.552}, "triton_wait": {"avg_ms": 68.592, "count": 124, "max_ms": 97.92, "p95_ms": 91.217}, "visualize": {"avg_ms": 10.747, "count": 124, "max_ms": 13.132, "p95_ms": 11.256}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 253}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3962.355, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:42:26+09:00", "last_error_summary": "[h264 @ 0x56057acc4d80] error while decoding MB 8 18, bytestream -26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:42:26+09:00", "last_error_summary": "[h264 @ 0x56057acc4d80] error while decoding MB 8 18, bytestream -26", "last_exit_at": "2026-07-01T16:42:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.474, "last_started_at": "2026-07-01T16:40:35+09:00", "pid": 9528, "process": {"exists": false, "pid": 9528}, "restart_count": 55, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.345, "sample_count": 1201, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30363, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:42:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:42:26+09:00", "last_frame_read_age_sec": 0.237, "last_frame_read_at": "2026-07-01T16:42:26+09:00", "mjpeg": {"age_sec": 0.227, "exists": true, "mtime": "2026-07-01T16:42:26.698127+09:00", "mtime_epoch": 1782891746.6981268, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192585}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:42:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 875.605, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.188, "pushed": 4.188, "read": 4.155}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.924, "count": 125, "max_ms": 10.997, "p95_ms": 9.514}, "gstreamer_push": {"avg_ms": 0.897, "count": 125, "max_ms": 2.811, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.542, "count": 125, "max_ms": 69.637, "p95_ms": 62.762}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.885, "count": 125, "max_ms": 97.099, "p95_ms": 88.454}, "yolo11_640_fire_11class": {"avg_ms": 55.33, "count": 125, "max_ms": 74.843, "p95_ms": 73.317}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.058, "count": 125, "max_ms": 97.908, "p95_ms": 91.207}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.244, "count": 125, "max_ms": 284.867, "p95_ms": 253.924}, "read_interval": {"avg_ms": 240.511, "count": 124, "max_ms": 294.952, "p95_ms": 262.97}, "triton_wait": {"avg_ms": 68.605, "count": 125, "max_ms": 97.92, "p95_ms": 91.217}, "visualize": {"avg_ms": 10.749, "count": 125, "max_ms": 13.132, "p95_ms": 11.256}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.844, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 253}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3962.348, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:42:26+09:00", "last_error_summary": "[h264 @ 0x56057acc4d80] error while decoding MB 8 18, bytestream -26", "last_exit_at": "2026-07-01T16:42:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.478, "last_started_at": "2026-07-01T16:40:35+09:00", "pid": 9528, "process": {"exists": false, "pid": 9528}, "restart_count": 55, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.345, "sample_count": 1201, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30363, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:42:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T16:42:26+09:00", "last_frame_read_age_sec": 0.241, "last_frame_read_at": "2026-07-01T16:42:26+09:00", "mjpeg": {"age_sec": 0.232, "exists": true, "mtime": "2026-07-01T16:42:26.698127+09:00", "mtime_epoch": 1782891746.6981268, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192585}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:42:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 875.605, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.188, "pushed": 4.188, "read": 4.154}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.924, "count": 125, "max_ms": 10.997, "p95_ms": 9.514}, "gstreamer_push": {"avg_ms": 0.897, "count": 125, "max_ms": 2.811, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.542, "count": 125, "max_ms": 69.637, "p95_ms": 62.762}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.885, "count": 125, "max_ms": 97.099, "p95_ms": 88.454}, "yolo11_640_fire_11class": {"avg_ms": 55.33, "count": 125, "max_ms": 74.843, "p95_ms": 73.317}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.058, "count": 125, "max_ms": 97.908, "p95_ms": 91.207}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.244, "count": 125, "max_ms": 284.867, "p95_ms": 253.924}, "read_interval": {"avg_ms": 240.511, "count": 124, "max_ms": 294.952, "p95_ms": 262.97}, "triton_wait": {"avg_ms": 68.605, "count": 125, "max_ms": 97.92, "p95_ms": 91.217}, "visualize": {"avg_ms": 10.749, "count": 125, "max_ms": 13.132, "p95_ms": 11.256}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.848, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 253}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4067.941, "limit_mb": 8796093022207.996}, "detail": {"pid": 9635, "restart_count": 56, "start_count": 57}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:42:26+09:00", "last_error_summary": "[h264 @ 0x56057acc4d80] error while decoding MB 8 18, bytestream -26", "last_exit_at": "2026-07-01T16:42:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:42:29+09:00", "pid": 9635, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.008, "name": "ffmpeg", "pid": 9635, "rss_mb": 139.008, "state": "S (sleeping)", "threads": 106}, "restart_count": 56, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.678, "sample_count": 1184, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30363, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:42:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.335, "last_frame_pushed_at": "2026-07-01T16:42:26+09:00", "last_frame_read_age_sec": 4.569, "last_frame_read_at": "2026-07-01T16:42:26+09:00", "mjpeg": {"age_sec": 4.319, "exists": true, "mtime": "2026-07-01T16:42:26.938134+09:00", "mtime_epoch": 1782891746.938134, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188155}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:42:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 875.605, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.582, "pushed": 3.582, "read": 3.548}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.976, "count": 106, "max_ms": 10.997, "p95_ms": 9.514}, "gstreamer_push": {"avg_ms": 0.895, "count": 107, "max_ms": 2.811, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.727, "count": 106, "max_ms": 69.637, "p95_ms": 62.762}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.069, "count": 106, "max_ms": 97.099, "p95_ms": 87.896}, "yolo11_640_fire_11class": {"avg_ms": 55.39, "count": 106, "max_ms": 74.843, "p95_ms": 72.991}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.0, "count": 106, "max_ms": 97.492, "p95_ms": 90.122}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.388, "count": 107, "max_ms": 284.867, "p95_ms": 252.474}, "read_interval": {"avg_ms": 240.786, "count": 106, "max_ms": 294.952, "p95_ms": 262.94}, "triton_wait": {"avg_ms": 68.598, "count": 106, "max_ms": 97.503, "p95_ms": 90.502}, "visualize": {"avg_ms": 10.743, "count": 106, "max_ms": 13.132, "p95_ms": 11.256}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.872, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 256}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3958.172, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:42:26+09:00", "last_error_summary": "[h264 @ 0x56057acc4d80] error while decoding MB 8 18, bytestream -26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:42:26+09:00", "last_error_summary": "[h264 @ 0x56057acc4d80] error while decoding MB 8 18, bytestream -26", "last_exit_at": "2026-07-01T16:44:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.006, "last_started_at": "2026-07-01T16:42:29+09:00", "pid": 9635, "process": {"exists": false, "pid": 9635}, "restart_count": 56, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 257.612, "sample_count": 1217, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30812, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:44:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T16:44:18+09:00", "last_frame_read_age_sec": 0.246, "last_frame_read_at": "2026-07-01T16:44:18+09:00", "mjpeg": {"age_sec": 0.237, "exists": true, "mtime": "2026-07-01T16:44:18.025309+09:00", "mtime_epoch": 1782891858.0253088, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184607}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:44:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 873.113, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.215, "pushed": 4.215, "read": 4.182}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.981, "count": 125, "max_ms": 13.523, "p95_ms": 9.726}, "gstreamer_push": {"avg_ms": 0.916, "count": 126, "max_ms": 2.813, "p95_ms": 1.73}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.469, "count": 125, "max_ms": 72.128, "p95_ms": 62.015}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.913, "count": 125, "max_ms": 90.382, "p95_ms": 86.183}, "yolo11_640_fire_11class": {"avg_ms": 54.736, "count": 125, "max_ms": 75.453, "p95_ms": 71.845}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.156, "count": 125, "max_ms": 96.514, "p95_ms": 92.821}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.625, "count": 126, "max_ms": 311.853, "p95_ms": 255.964}, "read_interval": {"avg_ms": 239.158, "count": 125, "max_ms": 321.903, "p95_ms": 265.191}, "triton_wait": {"avg_ms": 68.295, "count": 125, "max_ms": 96.526, "p95_ms": 92.832}, "visualize": {"avg_ms": 10.816, "count": 125, "max_ms": 15.726, "p95_ms": 11.847}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 256}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3958.289, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:42:26+09:00", "last_error_summary": "[h264 @ 0x56057acc4d80] error while decoding MB 8 18, bytestream -26", "last_exit_at": "2026-07-01T16:44:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.009, "last_started_at": "2026-07-01T16:42:29+09:00", "pid": 9635, "process": {"exists": false, "pid": 9635}, "restart_count": 56, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 257.612, "sample_count": 1217, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30812, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:44:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T16:44:18+09:00", "last_frame_read_age_sec": 0.25, "last_frame_read_at": "2026-07-01T16:44:18+09:00", "mjpeg": {"age_sec": 0.24, "exists": true, "mtime": "2026-07-01T16:44:18.025309+09:00", "mtime_epoch": 1782891858.0253088, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184607}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:44:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 873.113, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.215, "pushed": 4.215, "read": 4.181}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.981, "count": 125, "max_ms": 13.523, "p95_ms": 9.726}, "gstreamer_push": {"avg_ms": 0.916, "count": 126, "max_ms": 2.813, "p95_ms": 1.73}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.469, "count": 125, "max_ms": 72.128, "p95_ms": 62.015}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.913, "count": 125, "max_ms": 90.382, "p95_ms": 86.183}, "yolo11_640_fire_11class": {"avg_ms": 54.736, "count": 125, "max_ms": 75.453, "p95_ms": 71.845}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.156, "count": 125, "max_ms": 96.514, "p95_ms": 92.821}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.625, "count": 126, "max_ms": 311.853, "p95_ms": 255.964}, "read_interval": {"avg_ms": 239.158, "count": 125, "max_ms": 321.903, "p95_ms": 265.191}, "triton_wait": {"avg_ms": 68.295, "count": 125, "max_ms": 96.526, "p95_ms": 92.832}, "visualize": {"avg_ms": 10.816, "count": 125, "max_ms": 15.726, "p95_ms": 11.847}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 256}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4068.59, "limit_mb": 8796093022207.996}, "detail": {"pid": 9742, "restart_count": 57, "start_count": 58}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:42:26+09:00", "last_error_summary": "[h264 @ 0x56057acc4d80] error while decoding MB 8 18, bytestream -26", "last_exit_at": "2026-07-01T16:44:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:44:21+09:00", "pid": 9742, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.258, "name": "ffmpeg", "pid": 9742, "rss_mb": 139.258, "state": "S (sleeping)", "threads": 106}, "restart_count": 57, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 257.507, "sample_count": 1197, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 30812, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:44:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.779, "last_frame_pushed_at": "2026-07-01T16:44:18+09:00", "last_frame_read_age_sec": 5.021, "last_frame_read_at": "2026-07-01T16:44:18+09:00", "mjpeg": {"age_sec": 4.763, "exists": true, "mtime": "2026-07-01T16:44:18.273316+09:00", "mtime_epoch": 1782891858.2733157, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184674}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:44:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 873.113, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.522, "pushed": 3.522, "read": 3.488}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.987, "count": 105, "max_ms": 13.523, "p95_ms": 9.726}, "gstreamer_push": {"avg_ms": 0.917, "count": 105, "max_ms": 2.813, "p95_ms": 1.73}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.54, "count": 105, "max_ms": 72.128, "p95_ms": 62.015}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.07, "count": 105, "max_ms": 90.382, "p95_ms": 86.23}, "yolo11_640_fire_11class": {"avg_ms": 54.512, "count": 105, "max_ms": 75.434, "p95_ms": 71.266}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.093, "count": 105, "max_ms": 96.514, "p95_ms": 92.821}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.094, "count": 105, "max_ms": 311.853, "p95_ms": 258.197}, "read_interval": {"avg_ms": 240.58, "count": 104, "max_ms": 321.903, "p95_ms": 268.769}, "triton_wait": {"avg_ms": 68.256, "count": 105, "max_ms": 96.526, "p95_ms": 92.832}, "visualize": {"avg_ms": 10.844, "count": 105, "max_ms": 12.119, "p95_ms": 11.795}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.815, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 260}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3967.266, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564ffc0d8d00] error while decoding MB 47 14, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:46:11+09:00", "last_error_summary": "[h264 @ 0x564ffc0d8d00] error while decoding MB 47 14, bytestream -10", "last_exit_at": "2026-07-01T16:44:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.186, "last_started_at": "2026-07-01T16:44:21+09:00", "pid": 9742, "process": {"exists": false, "pid": 9742}, "restart_count": 57, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.48, "sample_count": 1207, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31254, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:46:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T16:46:11+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T16:46:11+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T16:46:11.180337+09:00", "mtime_epoch": 1782891971.180337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173922}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:46:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 876.262, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.106, "pushed": 4.106, "read": 4.106}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.533, "count": 122, "max_ms": 14.228, "p95_ms": 11.344}, "gstreamer_push": {"avg_ms": 0.905, "count": 123, "max_ms": 1.913, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.149, "count": 122, "max_ms": 68.171, "p95_ms": 62.349}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.414, "count": 122, "max_ms": 88.879, "p95_ms": 85.434}, "yolo11_640_fire_11class": {"avg_ms": 54.894, "count": 122, "max_ms": 78.337, "p95_ms": 71.737}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.212, "count": 122, "max_ms": 97.529, "p95_ms": 90.698}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.447, "count": 123, "max_ms": 330.941, "p95_ms": 256.695}, "read_interval": {"avg_ms": 245.084, "count": 123, "max_ms": 341.602, "p95_ms": 265.93}, "triton_wait": {"avg_ms": 68.36, "count": 122, "max_ms": 97.542, "p95_ms": 90.711}, "visualize": {"avg_ms": 11.211, "count": 122, "max_ms": 17.041, "p95_ms": 13.342}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 260}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3966.387, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:46:11+09:00", "last_error_summary": "[h264 @ 0x564ffc0d8d00] error while decoding MB 47 14, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:46:11+09:00", "last_error_summary": "[h264 @ 0x564ffc0d8d00] error while decoding MB 47 14, bytestream -10", "last_exit_at": "2026-07-01T16:46:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.364, "last_started_at": "2026-07-01T16:44:21+09:00", "pid": 9742, "process": {"exists": false, "pid": 9742}, "restart_count": 57, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.48, "sample_count": 1208, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31255, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:46:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:46:11+09:00", "last_frame_read_age_sec": 0.227, "last_frame_read_at": "2026-07-01T16:46:11+09:00", "mjpeg": {"age_sec": 0.219, "exists": true, "mtime": "2026-07-01T16:46:11.180337+09:00", "mtime_epoch": 1782891971.180337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173922}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:46:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 876.262, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.114, "pushed": 4.114, "read": 4.08}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.531, "count": 122, "max_ms": 14.228, "p95_ms": 11.344}, "gstreamer_push": {"avg_ms": 0.905, "count": 123, "max_ms": 1.913, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.039, "count": 122, "max_ms": 68.171, "p95_ms": 62.349}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.366, "count": 122, "max_ms": 88.879, "p95_ms": 85.434}, "yolo11_640_fire_11class": {"avg_ms": 54.859, "count": 122, "max_ms": 78.337, "p95_ms": 71.737}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.178, "count": 122, "max_ms": 97.529, "p95_ms": 90.698}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.327, "count": 123, "max_ms": 330.941, "p95_ms": 256.695}, "read_interval": {"avg_ms": 245.054, "count": 122, "max_ms": 341.602, "p95_ms": 265.93}, "triton_wait": {"avg_ms": 68.327, "count": 122, "max_ms": 97.542, "p95_ms": 90.711}, "visualize": {"avg_ms": 11.21, "count": 122, "max_ms": 17.041, "p95_ms": 13.342}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 260}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3966.379, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:46:11+09:00", "last_error_summary": "[h264 @ 0x564ffc0d8d00] error while decoding MB 47 14, bytestream -10", "last_exit_at": "2026-07-01T16:46:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.367, "last_started_at": "2026-07-01T16:44:21+09:00", "pid": 9742, "process": {"exists": false, "pid": 9742}, "restart_count": 57, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.48, "sample_count": 1208, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31255, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:46:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:46:11+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T16:46:11+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T16:46:11.180337+09:00", "mtime_epoch": 1782891971.180337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173922}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:46:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 876.262, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.113, "pushed": 4.113, "read": 4.08}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.531, "count": 122, "max_ms": 14.228, "p95_ms": 11.344}, "gstreamer_push": {"avg_ms": 0.905, "count": 123, "max_ms": 1.913, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.039, "count": 122, "max_ms": 68.171, "p95_ms": 62.349}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.366, "count": 122, "max_ms": 88.879, "p95_ms": 85.434}, "yolo11_640_fire_11class": {"avg_ms": 54.859, "count": 122, "max_ms": 78.337, "p95_ms": 71.737}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.178, "count": 122, "max_ms": 97.529, "p95_ms": 90.698}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.327, "count": 123, "max_ms": 330.941, "p95_ms": 256.695}, "read_interval": {"avg_ms": 245.054, "count": 122, "max_ms": 341.602, "p95_ms": 265.93}, "triton_wait": {"avg_ms": 68.327, "count": 122, "max_ms": 97.542, "p95_ms": 90.711}, "visualize": {"avg_ms": 11.21, "count": 122, "max_ms": 17.041, "p95_ms": 13.342}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 260}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4071.852, "limit_mb": 8796093022207.996}, "detail": {"pid": 9849, "restart_count": 58, "start_count": 59}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:46:11+09:00", "last_error_summary": "[h264 @ 0x564ffc0d8d00] error while decoding MB 47 14, bytestream -10", "last_exit_at": "2026-07-01T16:46:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:46:13+09:00", "pid": 9849, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.586, "name": "ffmpeg", "pid": 9849, "rss_mb": 136.586, "state": "S (sleeping)", "threads": 106}, "restart_count": 58, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.48, "sample_count": 1190, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31255, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:46:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.197, "last_frame_pushed_at": "2026-07-01T16:46:11+09:00", "last_frame_read_age_sec": 4.421, "last_frame_read_at": "2026-07-01T16:46:11+09:00", "mjpeg": {"age_sec": 4.181, "exists": true, "mtime": "2026-07-01T16:46:11.412343+09:00", "mtime_epoch": 1782891971.412343, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174175}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:46:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 876.262, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.542, "pushed": 3.542, "read": 3.509}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.415, "count": 105, "max_ms": 13.946, "p95_ms": 10.81}, "gstreamer_push": {"avg_ms": 0.902, "count": 106, "max_ms": 1.913, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.928, "count": 105, "max_ms": 68.171, "p95_ms": 62.349}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.349, "count": 105, "max_ms": 88.879, "p95_ms": 85.434}, "yolo11_640_fire_11class": {"avg_ms": 54.807, "count": 105, "max_ms": 78.337, "p95_ms": 72.074}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.118, "count": 105, "max_ms": 97.529, "p95_ms": 90.777}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.532, "count": 106, "max_ms": 330.941, "p95_ms": 256.695}, "read_interval": {"avg_ms": 245.257, "count": 105, "max_ms": 341.602, "p95_ms": 265.93}, "triton_wait": {"avg_ms": 68.288, "count": 105, "max_ms": 97.542, "p95_ms": 90.79}, "visualize": {"avg_ms": 11.15, "count": 105, "max_ms": 17.041, "p95_ms": 12.737}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 264}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3958.977, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:46:11+09:00", "last_error_summary": "[h264 @ 0x564ffc0d8d00] error while decoding MB 47 14, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:46:11+09:00", "last_error_summary": "[h264 @ 0x564ffc0d8d00] error while decoding MB 47 14, bytestream -10", "last_exit_at": "2026-07-01T16:48:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.342, "last_started_at": "2026-07-01T16:46:13+09:00", "pid": 9849, "process": {"exists": false, "pid": 9849}, "restart_count": 58, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.88, "sample_count": 1200, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31699, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:48:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:48:02+09:00", "last_frame_read_age_sec": 0.25, "last_frame_read_at": "2026-07-01T16:48:02+09:00", "mjpeg": {"age_sec": 0.239, "exists": true, "mtime": "2026-07-01T16:48:02.695158+09:00", "mtime_epoch": 1782892082.6951578, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171175}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:48:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 869.859, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.155, "pushed": 4.155, "read": 4.121}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.038, "count": 124, "max_ms": 12.219, "p95_ms": 10.007}, "gstreamer_push": {"avg_ms": 0.894, "count": 124, "max_ms": 2.976, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.418, "count": 124, "max_ms": 69.724, "p95_ms": 61.062}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.85, "count": 124, "max_ms": 98.115, "p95_ms": 93.35}, "yolo11_640_fire_11class": {"avg_ms": 55.409, "count": 124, "max_ms": 75.039, "p95_ms": 71.907}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.817, "count": 124, "max_ms": 96.253, "p95_ms": 92.223}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.22, "count": 124, "max_ms": 356.453, "p95_ms": 258.464}, "read_interval": {"avg_ms": 242.649, "count": 123, "max_ms": 367.828, "p95_ms": 267.874}, "triton_wait": {"avg_ms": 71.15, "count": 124, "max_ms": 98.128, "p95_ms": 93.76}, "visualize": {"avg_ms": 10.693, "count": 124, "max_ms": 13.922, "p95_ms": 11.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 264}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3958.969, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:46:11+09:00", "last_error_summary": "[h264 @ 0x564ffc0d8d00] error while decoding MB 47 14, bytestream -10", "last_exit_at": "2026-07-01T16:48:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.345, "last_started_at": "2026-07-01T16:46:13+09:00", "pid": 9849, "process": {"exists": false, "pid": 9849}, "restart_count": 58, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.88, "sample_count": 1200, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31699, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:48:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:48:02+09:00", "last_frame_read_age_sec": 0.253, "last_frame_read_at": "2026-07-01T16:48:02+09:00", "mjpeg": {"age_sec": 0.243, "exists": true, "mtime": "2026-07-01T16:48:02.695158+09:00", "mtime_epoch": 1782892082.6951578, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171175}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:48:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 869.859, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.154, "pushed": 4.154, "read": 4.121}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.038, "count": 124, "max_ms": 12.219, "p95_ms": 10.007}, "gstreamer_push": {"avg_ms": 0.894, "count": 124, "max_ms": 2.976, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.418, "count": 124, "max_ms": 69.724, "p95_ms": 61.062}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.85, "count": 124, "max_ms": 98.115, "p95_ms": 93.35}, "yolo11_640_fire_11class": {"avg_ms": 55.409, "count": 124, "max_ms": 75.039, "p95_ms": 71.907}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.817, "count": 124, "max_ms": 96.253, "p95_ms": 92.223}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.22, "count": 124, "max_ms": 356.453, "p95_ms": 258.464}, "read_interval": {"avg_ms": 242.649, "count": 123, "max_ms": 367.828, "p95_ms": 267.874}, "triton_wait": {"avg_ms": 71.15, "count": 124, "max_ms": 98.128, "p95_ms": 93.76}, "visualize": {"avg_ms": 10.693, "count": 124, "max_ms": 13.922, "p95_ms": 11.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.848, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 264}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4065.004, "limit_mb": 8796093022207.996}, "detail": {"pid": 9956, "restart_count": 59, "start_count": 60}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:46:11+09:00", "last_error_summary": "[h264 @ 0x564ffc0d8d00] error while decoding MB 47 14, bytestream -10", "last_exit_at": "2026-07-01T16:48:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:48:05+09:00", "pid": 9956, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.289, "name": "ffmpeg", "pid": 9956, "rss_mb": 139.289, "state": "S (sleeping)", "threads": 106}, "restart_count": 59, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 259.88, "sample_count": 1184, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 31699, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:48:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.098, "last_frame_pushed_at": "2026-07-01T16:48:02+09:00", "last_frame_read_age_sec": 4.345, "last_frame_read_at": "2026-07-01T16:48:02+09:00", "mjpeg": {"age_sec": 4.078, "exists": true, "mtime": "2026-07-01T16:48:02.951164+09:00", "mtime_epoch": 1782892082.951164, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171373}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:48:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 869.859, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.568, "pushed": 3.568, "read": 3.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.112, "count": 106, "max_ms": 12.219, "p95_ms": 10.255}, "gstreamer_push": {"avg_ms": 0.901, "count": 107, "max_ms": 2.976, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.681, "count": 106, "max_ms": 69.724, "p95_ms": 61.062}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.192, "count": 106, "max_ms": 98.115, "p95_ms": 93.93}, "yolo11_640_fire_11class": {"avg_ms": 55.881, "count": 106, "max_ms": 75.039, "p95_ms": 72.135}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.286, "count": 106, "max_ms": 96.253, "p95_ms": 92.416}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.688, "count": 107, "max_ms": 356.453, "p95_ms": 258.851}, "read_interval": {"avg_ms": 244.131, "count": 106, "max_ms": 367.828, "p95_ms": 267.919}, "triton_wait": {"avg_ms": 71.616, "count": 106, "max_ms": 98.128, "p95_ms": 93.943}, "visualize": {"avg_ms": 10.73, "count": 106, "max_ms": 13.922, "p95_ms": 11.432}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 267}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4195.094, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "last_exit_at": "2026-07-01T16:48:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.627, "last_started_at": "2026-07-01T16:48:05+09:00", "pid": 9956, "process": {"exists": false, "pid": 9956}, "restart_count": 59, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.643, "sample_count": 1200, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32131, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:49:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T16:49:53+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T16:49:53+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T16:49:53.609834+09:00", "mtime_epoch": 1782892193.6098342, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174918}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:49:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 871.363, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.068, "pushed": 4.068, "read": 4.068}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.56, "count": 121, "max_ms": 11.887, "p95_ms": 10.858}, "gstreamer_push": {"avg_ms": 0.957, "count": 122, "max_ms": 3.12, "p95_ms": 1.783}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.976, "count": 121, "max_ms": 70.588, "p95_ms": 65.344}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.503, "count": 121, "max_ms": 96.41, "p95_ms": 89.98}, "yolo11_640_fire_11class": {"avg_ms": 51.412, "count": 121, "max_ms": 74.085, "p95_ms": 70.509}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.714, "count": 121, "max_ms": 97.297, "p95_ms": 94.442}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.842, "count": 122, "max_ms": 314.599, "p95_ms": 271.001}, "read_interval": {"avg_ms": 247.334, "count": 122, "max_ms": 325.167, "p95_ms": 280.403}, "triton_wait": {"avg_ms": 67.506, "count": 121, "max_ms": 97.309, "p95_ms": 94.455}, "visualize": {"avg_ms": 11.109, "count": 121, "max_ms": 13.267, "p95_ms": 12.342}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 267}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4193.891, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "last_exit_at": "2026-07-01T16:49:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.804, "last_started_at": "2026-07-01T16:48:05+09:00", "pid": 9956, "process": {"exists": false, "pid": 9956}, "restart_count": 59, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.643, "sample_count": 1201, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:49:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:49:53+09:00", "last_frame_read_age_sec": 0.229, "last_frame_read_at": "2026-07-01T16:49:53+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T16:49:53.609834+09:00", "mtime_epoch": 1782892193.6098342, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174918}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:49:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 871.363, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.081, "pushed": 4.081, "read": 4.048}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.557, "count": 121, "max_ms": 11.887, "p95_ms": 10.858}, "gstreamer_push": {"avg_ms": 0.957, "count": 122, "max_ms": 3.12, "p95_ms": 1.783}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.889, "count": 121, "max_ms": 70.588, "p95_ms": 65.344}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.48, "count": 121, "max_ms": 96.41, "p95_ms": 89.98}, "yolo11_640_fire_11class": {"avg_ms": 51.335, "count": 121, "max_ms": 74.085, "p95_ms": 70.509}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.648, "count": 121, "max_ms": 97.297, "p95_ms": 94.442}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.766, "count": 122, "max_ms": 314.599, "p95_ms": 271.001}, "read_interval": {"avg_ms": 247.366, "count": 121, "max_ms": 325.167, "p95_ms": 280.403}, "triton_wait": {"avg_ms": 67.458, "count": 121, "max_ms": 97.309, "p95_ms": 94.455}, "visualize": {"avg_ms": 11.12, "count": 121, "max_ms": 13.267, "p95_ms": 12.342}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 267}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4193.883, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "last_exit_at": "2026-07-01T16:49:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.807, "last_started_at": "2026-07-01T16:48:05+09:00", "pid": 9956, "process": {"exists": false, "pid": 9956}, "restart_count": 59, "restart_count_per_hour": 29, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.643, "sample_count": 1201, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:49:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:49:53+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T16:49:53+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-01T16:49:53.609834+09:00", "mtime_epoch": 1782892193.6098342, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174918}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:49:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 871.363, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.081, "pushed": 4.081, "read": 4.047}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.557, "count": 121, "max_ms": 11.887, "p95_ms": 10.858}, "gstreamer_push": {"avg_ms": 0.957, "count": 122, "max_ms": 3.12, "p95_ms": 1.783}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.889, "count": 121, "max_ms": 70.588, "p95_ms": 65.344}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.48, "count": 121, "max_ms": 96.41, "p95_ms": 89.98}, "yolo11_640_fire_11class": {"avg_ms": 51.335, "count": 121, "max_ms": 74.085, "p95_ms": 70.509}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.648, "count": 121, "max_ms": 97.297, "p95_ms": 94.442}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.766, "count": 122, "max_ms": 314.599, "p95_ms": 271.001}, "read_interval": {"avg_ms": 247.366, "count": 121, "max_ms": 325.167, "p95_ms": 280.403}, "triton_wait": {"avg_ms": 67.458, "count": 121, "max_ms": 97.309, "p95_ms": 94.455}, "visualize": {"avg_ms": 11.12, "count": 121, "max_ms": 13.267, "p95_ms": 12.342}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 267}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4290.574, "limit_mb": 8796093022207.996}, "detail": {"pid": 10123, "restart_count": 60, "start_count": 61}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "last_exit_at": "2026-07-01T16:49:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:49:56+09:00", "pid": 10123, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.016, "name": "ffmpeg", "pid": 10123, "rss_mb": 142.016, "state": "S (sleeping)", "threads": 106}, "restart_count": 60, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.643, "sample_count": 1182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:49:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.721, "last_frame_pushed_at": "2026-07-01T16:49:53+09:00", "last_frame_read_age_sec": 4.948, "last_frame_read_at": "2026-07-01T16:49:53+09:00", "mjpeg": {"age_sec": 4.705, "exists": true, "mtime": "2026-07-01T16:49:53.845840+09:00", "mtime_epoch": 1782892193.8458397, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174803}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:49:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 871.363, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.448, "pushed": 3.448, "read": 3.414}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.493, "count": 102, "max_ms": 11.887, "p95_ms": 11.297}, "gstreamer_push": {"avg_ms": 0.968, "count": 103, "max_ms": 3.12, "p95_ms": 1.798}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.349, "count": 102, "max_ms": 70.588, "p95_ms": 65.344}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.694, "count": 102, "max_ms": 96.41, "p95_ms": 90.312}, "yolo11_640_fire_11class": {"avg_ms": 50.71, "count": 102, "max_ms": 74.085, "p95_ms": 68.34}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.047, "count": 102, "max_ms": 97.297, "p95_ms": 94.442}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.101, "count": 103, "max_ms": 314.599, "p95_ms": 278.5}, "read_interval": {"avg_ms": 246.653, "count": 102, "max_ms": 325.167, "p95_ms": 288.189}, "triton_wait": {"avg_ms": 67.006, "count": 102, "max_ms": 97.309, "p95_ms": 94.455}, "visualize": {"avg_ms": 11.13, "count": 102, "max_ms": 13.267, "p95_ms": 12.342}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 271}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3966.457, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "last_exit_at": "2026-07-01T16:51:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.387, "last_started_at": "2026-07-01T16:49:56+09:00", "pid": 10123, "process": {"exists": false, "pid": 10123}, "restart_count": 60, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.972, "sample_count": 1176, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32561, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:51:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:51:46+09:00", "last_frame_read_age_sec": 0.285, "last_frame_read_at": "2026-07-01T16:51:46+09:00", "mjpeg": {"age_sec": 0.277, "exists": true, "mtime": "2026-07-01T16:51:46.660456+09:00", "mtime_epoch": 1782892306.6604557, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179186}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:51:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 878.75, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.946, "pushed": 3.946, "read": 3.913}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.664, "count": 117, "max_ms": 14.032, "p95_ms": 10.686}, "gstreamer_push": {"avg_ms": 0.929, "count": 118, "max_ms": 2.028, "p95_ms": 1.632}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.356, "count": 117, "max_ms": 71.063, "p95_ms": 63.868}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.516, "count": 117, "max_ms": 93.519, "p95_ms": 85.579}, "yolo11_640_fire_11class": {"avg_ms": 55.947, "count": 117, "max_ms": 76.411, "p95_ms": 72.908}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.327, "count": 117, "max_ms": 95.744, "p95_ms": 90.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.146, "count": 118, "max_ms": 347.092, "p95_ms": 282.643}, "read_interval": {"avg_ms": 255.06, "count": 117, "max_ms": 356.243, "p95_ms": 294.725}, "triton_wait": {"avg_ms": 69.601, "count": 117, "max_ms": 95.757, "p95_ms": 90.512}, "visualize": {"avg_ms": 11.431, "count": 117, "max_ms": 15.12, "p95_ms": 12.471}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 271}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3966.449, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "last_exit_at": "2026-07-01T16:51:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.391, "last_started_at": "2026-07-01T16:49:56+09:00", "pid": 10123, "process": {"exists": false, "pid": 10123}, "restart_count": 60, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.972, "sample_count": 1176, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32561, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:51:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T16:51:46+09:00", "last_frame_read_age_sec": 0.288, "last_frame_read_at": "2026-07-01T16:51:46+09:00", "mjpeg": {"age_sec": 0.28, "exists": true, "mtime": "2026-07-01T16:51:46.660456+09:00", "mtime_epoch": 1782892306.6604557, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179186}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:51:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 878.75, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.946, "pushed": 3.946, "read": 3.912}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.664, "count": 117, "max_ms": 14.032, "p95_ms": 10.686}, "gstreamer_push": {"avg_ms": 0.929, "count": 118, "max_ms": 2.028, "p95_ms": 1.632}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.356, "count": 117, "max_ms": 71.063, "p95_ms": 63.868}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.516, "count": 117, "max_ms": 93.519, "p95_ms": 85.579}, "yolo11_640_fire_11class": {"avg_ms": 55.947, "count": 117, "max_ms": 76.411, "p95_ms": 72.908}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.327, "count": 117, "max_ms": 95.744, "p95_ms": 90.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.146, "count": 118, "max_ms": 347.092, "p95_ms": 282.643}, "read_interval": {"avg_ms": 255.06, "count": 117, "max_ms": 356.243, "p95_ms": 294.725}, "triton_wait": {"avg_ms": 69.601, "count": 117, "max_ms": 95.757, "p95_ms": 90.512}, "visualize": {"avg_ms": 11.431, "count": 117, "max_ms": 15.12, "p95_ms": 12.471}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 271}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4082.105, "limit_mb": 8796093022207.996}, "detail": {"pid": 10230, "restart_count": 61, "start_count": 62}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "last_exit_at": "2026-07-01T16:51:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:51:49+09:00", "pid": 10230, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.133, "name": "ffmpeg", "pid": 10230, "rss_mb": 142.133, "state": "S (sleeping)", "threads": 106}, "restart_count": 61, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.04, "sample_count": 1159, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 32561, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:51:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.118, "last_frame_pushed_at": "2026-07-01T16:51:46+09:00", "last_frame_read_age_sec": 4.4, "last_frame_read_at": "2026-07-01T16:51:46+09:00", "mjpeg": {"age_sec": 4.104, "exists": true, "mtime": "2026-07-01T16:51:46.948462+09:00", "mtime_epoch": 1782892306.9484622, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:51:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 878.75, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.378, "pushed": 3.378, "read": 3.345}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.707, "count": 100, "max_ms": 14.032, "p95_ms": 10.621}, "gstreamer_push": {"avg_ms": 0.936, "count": 101, "max_ms": 2.028, "p95_ms": 1.632}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.185, "count": 100, "max_ms": 71.063, "p95_ms": 63.868}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.633, "count": 100, "max_ms": 93.519, "p95_ms": 85.579}, "yolo11_640_fire_11class": {"avg_ms": 56.089, "count": 100, "max_ms": 76.411, "p95_ms": 72.908}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.578, "count": 100, "max_ms": 95.744, "p95_ms": 90.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.41, "count": 101, "max_ms": 347.092, "p95_ms": 282.643}, "read_interval": {"avg_ms": 257.425, "count": 100, "max_ms": 356.243, "p95_ms": 294.725}, "triton_wait": {"avg_ms": 69.896, "count": 100, "max_ms": 95.757, "p95_ms": 90.512}, "visualize": {"avg_ms": 11.463, "count": 100, "max_ms": 15.12, "p95_ms": 12.393}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 275}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3972.496, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "last_exit_at": "2026-07-01T16:53:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.234, "last_started_at": "2026-07-01T16:51:49+09:00", "pid": 10230, "process": {"exists": false, "pid": 10230}, "restart_count": 61, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.408, "sample_count": 1166, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33002, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:53:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:53:41+09:00", "last_frame_read_age_sec": 0.243, "last_frame_read_at": "2026-07-01T16:53:41+09:00", "mjpeg": {"age_sec": 0.234, "exists": true, "mtime": "2026-07-01T16:53:41.051020+09:00", "mtime_epoch": 1782892421.0510204, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175006}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:53:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 876.895, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.017, "pushed": 4.017, "read": 3.983}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.821, "count": 119, "max_ms": 17.868, "p95_ms": 11.037}, "gstreamer_push": {"avg_ms": 0.931, "count": 120, "max_ms": 2.885, "p95_ms": 1.175}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.999, "count": 119, "max_ms": 71.345, "p95_ms": 67.363}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.746, "count": 119, "max_ms": 88.813, "p95_ms": 85.468}, "yolo11_640_fire_11class": {"avg_ms": 52.685, "count": 119, "max_ms": 72.392, "p95_ms": 69.677}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.17, "count": 119, "max_ms": 91.295, "p95_ms": 87.835}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.256, "count": 120, "max_ms": 333.85, "p95_ms": 272.461}, "read_interval": {"avg_ms": 250.948, "count": 119, "max_ms": 343.79, "p95_ms": 284.365}, "triton_wait": {"avg_ms": 66.228, "count": 119, "max_ms": 91.308, "p95_ms": 88.513}, "visualize": {"avg_ms": 11.334, "count": 119, "max_ms": 14.427, "p95_ms": 12.377}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 275}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3972.484, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "last_exit_at": "2026-07-01T16:53:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.237, "last_started_at": "2026-07-01T16:51:49+09:00", "pid": 10230, "process": {"exists": false, "pid": 10230}, "restart_count": 61, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.408, "sample_count": 1166, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33002, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:53:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:53:41+09:00", "last_frame_read_age_sec": 0.247, "last_frame_read_at": "2026-07-01T16:53:41+09:00", "mjpeg": {"age_sec": 0.238, "exists": true, "mtime": "2026-07-01T16:53:41.051020+09:00", "mtime_epoch": 1782892421.0510204, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175006}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:53:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 876.895, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.016, "pushed": 4.016, "read": 3.983}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.821, "count": 119, "max_ms": 17.868, "p95_ms": 11.037}, "gstreamer_push": {"avg_ms": 0.931, "count": 120, "max_ms": 2.885, "p95_ms": 1.175}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.999, "count": 119, "max_ms": 71.345, "p95_ms": 67.363}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.746, "count": 119, "max_ms": 88.813, "p95_ms": 85.468}, "yolo11_640_fire_11class": {"avg_ms": 52.685, "count": 119, "max_ms": 72.392, "p95_ms": 69.677}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.17, "count": 119, "max_ms": 91.295, "p95_ms": 87.835}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.256, "count": 120, "max_ms": 333.85, "p95_ms": 272.461}, "read_interval": {"avg_ms": 250.948, "count": 119, "max_ms": 343.79, "p95_ms": 284.365}, "triton_wait": {"avg_ms": 66.228, "count": 119, "max_ms": 91.308, "p95_ms": 88.513}, "visualize": {"avg_ms": 11.334, "count": 119, "max_ms": 14.427, "p95_ms": 12.377}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 275}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4078.16, "limit_mb": 8796093022207.996}, "detail": {"pid": 10337, "restart_count": 62, "start_count": 63}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:49:53+09:00", "last_error_summary": "[h264 @ 0x55c8b9603b80] error while decoding MB 76 38, bytestream -8", "last_exit_at": "2026-07-01T16:53:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:53:44+09:00", "pid": 10337, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.109, "name": "ffmpeg", "pid": 10337, "rss_mb": 139.109, "state": "S (sleeping)", "threads": 106}, "restart_count": 62, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.934, "sample_count": 1147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33002, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:53:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.776, "last_frame_pushed_at": "2026-07-01T16:53:41+09:00", "last_frame_read_age_sec": 5.016, "last_frame_read_at": "2026-07-01T16:53:41+09:00", "mjpeg": {"age_sec": 4.759, "exists": true, "mtime": "2026-07-01T16:53:41.299026+09:00", "mtime_epoch": 1782892421.299026, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175187}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:53:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.27, "name": "celery", "pid": 237, "rss_mb": 876.895, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.353, "pushed": 3.353, "read": 3.32}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.994, "count": 100, "max_ms": 17.868, "p95_ms": 11.037}, "gstreamer_push": {"avg_ms": 0.944, "count": 100, "max_ms": 2.885, "p95_ms": 1.795}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.766, "count": 100, "max_ms": 71.345, "p95_ms": 67.363}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.027, "count": 100, "max_ms": 88.813, "p95_ms": 85.468}, "yolo11_640_fire_11class": {"avg_ms": 52.715, "count": 100, "max_ms": 72.392, "p95_ms": 69.677}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.442, "count": 100, "max_ms": 91.295, "p95_ms": 87.835}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.16, "count": 100, "max_ms": 333.85, "p95_ms": 273.436}, "read_interval": {"avg_ms": 252.893, "count": 99, "max_ms": 343.79, "p95_ms": 314.8}, "triton_wait": {"avg_ms": 66.508, "count": 100, "max_ms": 91.308, "p95_ms": 88.513}, "visualize": {"avg_ms": 11.348, "count": 100, "max_ms": 14.427, "p95_ms": 12.367}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.821, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 278}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3963.52, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5592e7041a00] error while decoding MB 15 39, bytestream -34", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:55:33+09:00", "last_error_summary": "[h264 @ 0x5592e7041a00] error while decoding MB 15 39, bytestream -34", "last_exit_at": "2026-07-01T16:53:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.899, "last_started_at": "2026-07-01T16:53:44+09:00", "pid": 10337, "process": {"exists": false, "pid": 10337}, "restart_count": 62, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.81, "sample_count": 1167, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33442, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:55:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T16:55:33+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T16:55:33+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T16:55:33.913481+09:00", "mtime_epoch": 1782892533.9134815, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175705}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:55:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.453, "name": "celery", "pid": 237, "rss_mb": 871.77, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.067, "pushed": 4.067, "read": 4.101}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.455, "count": 122, "max_ms": 13.748, "p95_ms": 10.49}, "gstreamer_push": {"avg_ms": 0.933, "count": 122, "max_ms": 1.947, "p95_ms": 1.782}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.362, "count": 122, "max_ms": 74.867, "p95_ms": 61.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.601, "count": 122, "max_ms": 95.661, "p95_ms": 81.518}, "yolo11_640_fire_11class": {"avg_ms": 53.713, "count": 122, "max_ms": 77.887, "p95_ms": 69.414}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.63, "count": 122, "max_ms": 98.636, "p95_ms": 86.85}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.695, "count": 122, "max_ms": 292.106, "p95_ms": 275.584}, "read_interval": {"avg_ms": 245.349, "count": 123, "max_ms": 302.285, "p95_ms": 285.523}, "triton_wait": {"avg_ms": 66.656, "count": 122, "max_ms": 98.649, "p95_ms": 86.863}, "visualize": {"avg_ms": 11.236, "count": 122, "max_ms": 16.051, "p95_ms": 12.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 278}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3965.625, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:55:33+09:00", "last_error_summary": "[h264 @ 0x5592e7041a00] error while decoding MB 15 39, bytestream -34", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:55:33+09:00", "last_error_summary": "[h264 @ 0x5592e7041a00] error while decoding MB 15 39, bytestream -34", "last_exit_at": "2026-07-01T16:55:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.094, "last_started_at": "2026-07-01T16:53:44+09:00", "pid": 10337, "process": {"exists": false, "pid": 10337}, "restart_count": 62, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.81, "sample_count": 1168, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33443, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:55:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:55:34+09:00", "last_frame_read_age_sec": 0.248, "last_frame_read_at": "2026-07-01T16:55:33+09:00", "mjpeg": {"age_sec": 0.238, "exists": true, "mtime": "2026-07-01T16:55:33.913481+09:00", "mtime_epoch": 1782892533.9134815, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175705}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:55:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.453, "name": "celery", "pid": 237, "rss_mb": 871.77, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.105, "pushed": 4.105, "read": 4.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.461, "count": 122, "max_ms": 13.748, "p95_ms": 10.49}, "gstreamer_push": {"avg_ms": 0.932, "count": 123, "max_ms": 1.947, "p95_ms": 1.782}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.28, "count": 122, "max_ms": 74.867, "p95_ms": 61.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.59, "count": 122, "max_ms": 95.661, "p95_ms": 81.518}, "yolo11_640_fire_11class": {"avg_ms": 53.705, "count": 122, "max_ms": 77.887, "p95_ms": 69.414}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.624, "count": 122, "max_ms": 98.636, "p95_ms": 86.85}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.761, "count": 123, "max_ms": 292.106, "p95_ms": 275.584}, "read_interval": {"avg_ms": 245.426, "count": 122, "max_ms": 302.285, "p95_ms": 285.523}, "triton_wait": {"avg_ms": 66.65, "count": 122, "max_ms": 98.649, "p95_ms": 86.863}, "visualize": {"avg_ms": 11.243, "count": 122, "max_ms": 16.051, "p95_ms": 12.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 278}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3965.617, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:55:33+09:00", "last_error_summary": "[h264 @ 0x5592e7041a00] error while decoding MB 15 39, bytestream -34", "last_exit_at": "2026-07-01T16:55:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.097, "last_started_at": "2026-07-01T16:53:44+09:00", "pid": 10337, "process": {"exists": false, "pid": 10337}, "restart_count": 62, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.81, "sample_count": 1168, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33443, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:55:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:55:34+09:00", "last_frame_read_age_sec": 0.251, "last_frame_read_at": "2026-07-01T16:55:33+09:00", "mjpeg": {"age_sec": 0.241, "exists": true, "mtime": "2026-07-01T16:55:33.913481+09:00", "mtime_epoch": 1782892533.9134815, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175705}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:55:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.453, "name": "celery", "pid": 237, "rss_mb": 871.77, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.104, "pushed": 4.104, "read": 4.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.461, "count": 122, "max_ms": 13.748, "p95_ms": 10.49}, "gstreamer_push": {"avg_ms": 0.932, "count": 123, "max_ms": 1.947, "p95_ms": 1.782}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.28, "count": 122, "max_ms": 74.867, "p95_ms": 61.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.59, "count": 122, "max_ms": 95.661, "p95_ms": 81.518}, "yolo11_640_fire_11class": {"avg_ms": 53.705, "count": 122, "max_ms": 77.887, "p95_ms": 69.414}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.624, "count": 122, "max_ms": 98.636, "p95_ms": 86.85}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.761, "count": 123, "max_ms": 292.106, "p95_ms": 275.584}, "read_interval": {"avg_ms": 245.426, "count": 122, "max_ms": 302.285, "p95_ms": 285.523}, "triton_wait": {"avg_ms": 66.65, "count": 122, "max_ms": 98.649, "p95_ms": 86.863}, "visualize": {"avg_ms": 11.243, "count": 122, "max_ms": 16.051, "p95_ms": 12.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 278}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4073.5, "limit_mb": 8796093022207.996}, "detail": {"pid": 10444, "restart_count": 63, "start_count": 64}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:55:33+09:00", "last_error_summary": "[h264 @ 0x5592e7041a00] error while decoding MB 15 39, bytestream -34", "last_exit_at": "2026-07-01T16:55:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:55:36+09:00", "pid": 10444, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.203, "name": "ffmpeg", "pid": 10444, "rss_mb": 139.203, "state": "S (sleeping)", "threads": 106}, "restart_count": 63, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.455, "sample_count": 1148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33443, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:55:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.851, "last_frame_pushed_at": "2026-07-01T16:55:34+09:00", "last_frame_read_age_sec": 5.097, "last_frame_read_at": "2026-07-01T16:55:33+09:00", "mjpeg": {"age_sec": 4.834, "exists": true, "mtime": "2026-07-01T16:55:34.165487+09:00", "mtime_epoch": 1782892534.165487, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175413}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:55:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.453, "name": "celery", "pid": 237, "rss_mb": 871.77, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.437, "pushed": 3.437, "read": 3.404}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.468, "count": 102, "max_ms": 13.748, "p95_ms": 10.49}, "gstreamer_push": {"avg_ms": 0.922, "count": 103, "max_ms": 1.947, "p95_ms": 1.259}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.846, "count": 102, "max_ms": 63.981, "p95_ms": 61.025}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.37, "count": 102, "max_ms": 88.813, "p95_ms": 81.518}, "yolo11_640_fire_11class": {"avg_ms": 53.519, "count": 102, "max_ms": 74.099, "p95_ms": 69.414}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.609, "count": 102, "max_ms": 93.309, "p95_ms": 86.85}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.48, "count": 103, "max_ms": 286.086, "p95_ms": 275.584}, "read_interval": {"avg_ms": 246.155, "count": 102, "max_ms": 297.317, "p95_ms": 285.523}, "triton_wait": {"avg_ms": 66.629, "count": 102, "max_ms": 93.321, "p95_ms": 86.863}, "visualize": {"avg_ms": 11.294, "count": 102, "max_ms": 16.051, "p95_ms": 12.862}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 282}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3969.875, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e47f20d500] error while decoding MB 0 34, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:57:26+09:00", "last_error_summary": "[h264 @ 0x55e47f20d500] error while decoding MB 0 34, bytestream -5", "last_exit_at": "2026-07-01T16:55:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.572, "last_started_at": "2026-07-01T16:55:36+09:00", "pid": 10444, "process": {"exists": false, "pid": 10444}, "restart_count": 63, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.668, "sample_count": 1180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33881, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:57:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T16:57:26+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T16:57:26+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T16:57:26.531882+09:00", "mtime_epoch": 1782892646.5318823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:57:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.453, "name": "celery", "pid": 237, "rss_mb": 877.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.057, "pushed": 4.057, "read": 4.057}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.647, "count": 121, "max_ms": 14.463, "p95_ms": 12.02}, "gstreamer_push": {"avg_ms": 0.907, "count": 121, "max_ms": 2.837, "p95_ms": 0.991}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.09, "count": 121, "max_ms": 73.534, "p95_ms": 62.574}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.566, "count": 121, "max_ms": 88.801, "p95_ms": 86.055}, "yolo11_640_fire_11class": {"avg_ms": 53.28, "count": 121, "max_ms": 74.507, "p95_ms": 69.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.065, "count": 121, "max_ms": 96.236, "p95_ms": 89.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.273, "count": 121, "max_ms": 304.234, "p95_ms": 267.858}, "read_interval": {"avg_ms": 247.902, "count": 121, "max_ms": 315.971, "p95_ms": 278.565}, "triton_wait": {"avg_ms": 67.376, "count": 121, "max_ms": 96.25, "p95_ms": 89.317}, "visualize": {"avg_ms": 11.515, "count": 121, "max_ms": 16.687, "p95_ms": 14.298}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.824, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 282}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3968.582, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:57:26+09:00", "last_error_summary": "[h264 @ 0x55e47f20d500] error while decoding MB 0 34, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:57:26+09:00", "last_error_summary": "[h264 @ 0x55e47f20d500] error while decoding MB 0 34, bytestream -5", "last_exit_at": "2026-07-01T16:57:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.771, "last_started_at": "2026-07-01T16:55:36+09:00", "pid": 10444, "process": {"exists": false, "pid": 10444}, "restart_count": 63, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.668, "sample_count": 1181, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:57:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:57:26+09:00", "last_frame_read_age_sec": 0.25, "last_frame_read_at": "2026-07-01T16:57:26+09:00", "mjpeg": {"age_sec": 0.237, "exists": true, "mtime": "2026-07-01T16:57:26.531882+09:00", "mtime_epoch": 1782892646.5318823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:57:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.453, "name": "celery", "pid": 237, "rss_mb": 877.816, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.061, "pushed": 4.061, "read": 4.027}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.658, "count": 121, "max_ms": 14.463, "p95_ms": 12.02}, "gstreamer_push": {"avg_ms": 0.907, "count": 121, "max_ms": 2.837, "p95_ms": 0.991}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.994, "count": 121, "max_ms": 73.534, "p95_ms": 62.574}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.525, "count": 121, "max_ms": 88.801, "p95_ms": 86.055}, "yolo11_640_fire_11class": {"avg_ms": 53.235, "count": 121, "max_ms": 74.507, "p95_ms": 69.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.056, "count": 121, "max_ms": 96.236, "p95_ms": 89.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.452, "count": 121, "max_ms": 304.234, "p95_ms": 267.858}, "read_interval": {"avg_ms": 248.022, "count": 120, "max_ms": 315.971, "p95_ms": 278.565}, "triton_wait": {"avg_ms": 67.367, "count": 121, "max_ms": 96.25, "p95_ms": 89.317}, "visualize": {"avg_ms": 11.512, "count": 121, "max_ms": 16.687, "p95_ms": 14.298}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.798, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 282}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3968.574, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:57:26+09:00", "last_error_summary": "[h264 @ 0x55e47f20d500] error while decoding MB 0 34, bytestream -5", "last_exit_at": "2026-07-01T16:57:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.774, "last_started_at": "2026-07-01T16:55:36+09:00", "pid": 10444, "process": {"exists": false, "pid": 10444}, "restart_count": 63, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.668, "sample_count": 1181, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:57:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:57:26+09:00", "last_frame_read_age_sec": 0.254, "last_frame_read_at": "2026-07-01T16:57:26+09:00", "mjpeg": {"age_sec": 0.241, "exists": true, "mtime": "2026-07-01T16:57:26.531882+09:00", "mtime_epoch": 1782892646.5318823, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:57:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.453, "name": "celery", "pid": 237, "rss_mb": 877.816, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.06, "pushed": 4.06, "read": 4.027}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.658, "count": 121, "max_ms": 14.463, "p95_ms": 12.02}, "gstreamer_push": {"avg_ms": 0.907, "count": 121, "max_ms": 2.837, "p95_ms": 0.991}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.994, "count": 121, "max_ms": 73.534, "p95_ms": 62.574}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.525, "count": 121, "max_ms": 88.801, "p95_ms": 86.055}, "yolo11_640_fire_11class": {"avg_ms": 53.235, "count": 121, "max_ms": 74.507, "p95_ms": 69.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.056, "count": 121, "max_ms": 96.236, "p95_ms": 89.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.452, "count": 121, "max_ms": 304.234, "p95_ms": 267.858}, "read_interval": {"avg_ms": 248.022, "count": 120, "max_ms": 315.971, "p95_ms": 278.565}, "triton_wait": {"avg_ms": 67.367, "count": 121, "max_ms": 96.25, "p95_ms": 89.317}, "visualize": {"avg_ms": 11.512, "count": 121, "max_ms": 16.687, "p95_ms": 14.298}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.801, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 282}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4073.23, "limit_mb": 8796093022207.996}, "detail": {"pid": 10551, "restart_count": 64, "start_count": 65}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:57:26+09:00", "last_error_summary": "[h264 @ 0x55e47f20d500] error while decoding MB 0 34, bytestream -5", "last_exit_at": "2026-07-01T16:57:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:57:29+09:00", "pid": 10551, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.781, "name": "ffmpeg", "pid": 10551, "rss_mb": 136.781, "state": "S (sleeping)", "threads": 106}, "restart_count": 64, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.616, "sample_count": 1163, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 33882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:57:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.366, "last_frame_pushed_at": "2026-07-01T16:57:26+09:00", "last_frame_read_age_sec": 4.614, "last_frame_read_at": "2026-07-01T16:57:26+09:00", "mjpeg": {"age_sec": 4.349, "exists": true, "mtime": "2026-07-01T16:57:26.783888+09:00", "mtime_epoch": 1782892646.7838876, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174793}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:57:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.453, "name": "celery", "pid": 237, "rss_mb": 877.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.481, "pushed": 3.481, "read": 3.447}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.67, "count": 103, "max_ms": 14.463, "p95_ms": 12.017}, "gstreamer_push": {"avg_ms": 0.915, "count": 104, "max_ms": 2.837, "p95_ms": 0.996}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.974, "count": 103, "max_ms": 73.534, "p95_ms": 62.574}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.067, "count": 103, "max_ms": 88.448, "p95_ms": 85.226}, "yolo11_640_fire_11class": {"avg_ms": 53.051, "count": 103, "max_ms": 74.507, "p95_ms": 68.6}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.836, "count": 103, "max_ms": 96.236, "p95_ms": 89.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.922, "count": 104, "max_ms": 304.234, "p95_ms": 267.556}, "read_interval": {"avg_ms": 247.488, "count": 103, "max_ms": 315.971, "p95_ms": 276.572}, "triton_wait": {"avg_ms": 67.058, "count": 103, "max_ms": 96.25, "p95_ms": 89.317}, "visualize": {"avg_ms": 11.555, "count": 103, "max_ms": 16.687, "p95_ms": 14.298}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 285}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3968.852, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b09ee09e80] error while decoding MB 48 30, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 55, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:59:18+09:00", "last_error_summary": "[h264 @ 0x55b09ee09e80] error while decoding MB 48 30, bytestream -11", "last_exit_at": "2026-07-01T16:57:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.876, "last_started_at": "2026-07-01T16:57:29+09:00", "pid": 10551, "process": {"exists": false, "pid": 10551}, "restart_count": 64, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.079, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34316, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:59:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T16:59:17+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T16:59:17+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T16:59:17.966214+09:00", "mtime_epoch": 1782892757.9662142, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176093}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:59:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.836, "name": "celery", "pid": 237, "rss_mb": 875.367, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.167, "pushed": 4.167, "read": 4.167}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.999, "count": 124, "max_ms": 13.293, "p95_ms": 9.896}, "gstreamer_push": {"avg_ms": 0.929, "count": 124, "max_ms": 2.837, "p95_ms": 1.792}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.393, "count": 124, "max_ms": 69.949, "p95_ms": 61.629}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.143, "count": 124, "max_ms": 89.411, "p95_ms": 83.839}, "yolo11_640_fire_11class": {"avg_ms": 52.914, "count": 124, "max_ms": 70.488, "p95_ms": 67.987}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.19, "count": 124, "max_ms": 92.026, "p95_ms": 88.28}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.911, "count": 124, "max_ms": 295.498, "p95_ms": 254.832}, "read_interval": {"avg_ms": 241.539, "count": 124, "max_ms": 304.741, "p95_ms": 265.433}, "triton_wait": {"avg_ms": 67.375, "count": 124, "max_ms": 92.037, "p95_ms": 88.293}, "visualize": {"avg_ms": 10.828, "count": 124, "max_ms": 11.919, "p95_ms": 11.343}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.761, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 285}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3967.926, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T16:59:18+09:00", "last_error_summary": "[h264 @ 0x55b09ee09e80] error while decoding MB 48 30, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 55, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:59:18+09:00", "last_error_summary": "[h264 @ 0x55b09ee09e80] error while decoding MB 48 30, bytestream -11", "last_exit_at": "2026-07-01T16:59:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.057, "last_started_at": "2026-07-01T16:57:29+09:00", "pid": 10551, "process": {"exists": false, "pid": 10551}, "restart_count": 64, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.079, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34317, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:59:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T16:59:18+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T16:59:17+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T16:59:17.966214+09:00", "mtime_epoch": 1782892757.9662142, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176093}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:59:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.836, "name": "celery", "pid": 237, "rss_mb": 875.367, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.175, "pushed": 4.175, "read": 4.141}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.967, "count": 124, "max_ms": 10.622, "p95_ms": 9.824}, "gstreamer_push": {"avg_ms": 0.928, "count": 125, "max_ms": 2.837, "p95_ms": 1.792}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.284, "count": 124, "max_ms": 69.949, "p95_ms": 61.629}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.024, "count": 124, "max_ms": 89.411, "p95_ms": 83.839}, "yolo11_640_fire_11class": {"avg_ms": 52.816, "count": 124, "max_ms": 70.488, "p95_ms": 67.987}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.071, "count": 124, "max_ms": 92.026, "p95_ms": 88.28}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.879, "count": 125, "max_ms": 295.498, "p95_ms": 254.832}, "read_interval": {"avg_ms": 241.539, "count": 124, "max_ms": 304.741, "p95_ms": 265.433}, "triton_wait": {"avg_ms": 67.256, "count": 124, "max_ms": 92.037, "p95_ms": 88.293}, "visualize": {"avg_ms": 10.837, "count": 124, "max_ms": 11.919, "p95_ms": 11.343}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 285}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3968.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 55, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:59:18+09:00", "last_error_summary": "[h264 @ 0x55b09ee09e80] error while decoding MB 48 30, bytestream -11", "last_exit_at": "2026-07-01T16:59:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.061, "last_started_at": "2026-07-01T16:57:29+09:00", "pid": 10551, "process": {"exists": false, "pid": 10551}, "restart_count": 64, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.079, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34317, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:59:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T16:59:18+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T16:59:17+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-01T16:59:17.966214+09:00", "mtime_epoch": 1782892757.9662142, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176093}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:59:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.836, "name": "celery", "pid": 237, "rss_mb": 875.367, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.174, "pushed": 4.174, "read": 4.141}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.967, "count": 124, "max_ms": 10.622, "p95_ms": 9.824}, "gstreamer_push": {"avg_ms": 0.928, "count": 125, "max_ms": 2.837, "p95_ms": 1.792}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.284, "count": 124, "max_ms": 69.949, "p95_ms": 61.629}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.024, "count": 124, "max_ms": 89.411, "p95_ms": 83.839}, "yolo11_640_fire_11class": {"avg_ms": 52.816, "count": 124, "max_ms": 70.488, "p95_ms": 67.987}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.071, "count": 124, "max_ms": 92.026, "p95_ms": 88.28}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.879, "count": 125, "max_ms": 295.498, "p95_ms": 254.832}, "read_interval": {"avg_ms": 241.539, "count": 124, "max_ms": 304.741, "p95_ms": 265.433}, "triton_wait": {"avg_ms": 67.256, "count": 124, "max_ms": 92.037, "p95_ms": 88.293}, "visualize": {"avg_ms": 10.837, "count": 124, "max_ms": 11.919, "p95_ms": 11.343}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 285}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3988.734, "limit_mb": 8796093022207.996}, "detail": {"pid": 10658, "restart_count": 65, "start_count": 66}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 55, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T16:59:18+09:00", "last_error_summary": "[h264 @ 0x55b09ee09e80] error while decoding MB 48 30, bytestream -11", "last_exit_at": "2026-07-01T16:59:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T16:59:20+09:00", "pid": 10658, "process": {"exists": true, "fd_count": 4, "hwm_mb": 52.637, "name": "ffmpeg", "pid": 10658, "rss_mb": 52.637, "state": "S (sleeping)", "threads": 1}, "restart_count": 65, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.079, "sample_count": 1168, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34317, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T16:59:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.184, "last_frame_pushed_at": "2026-07-01T16:59:18+09:00", "last_frame_read_age_sec": 4.414, "last_frame_read_at": "2026-07-01T16:59:17+09:00", "mjpeg": {"age_sec": 4.169, "exists": true, "mtime": "2026-07-01T16:59:18.202219+09:00", "mtime_epoch": 1782892758.2022192, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176204}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T16:59:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.836, "name": "celery", "pid": 237, "rss_mb": 875.367, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.61, "pushed": 3.61, "read": 3.576}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.964, "count": 107, "max_ms": 10.524, "p95_ms": 9.632}, "gstreamer_push": {"avg_ms": 0.917, "count": 108, "max_ms": 2.837, "p95_ms": 0.977}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.01, "count": 107, "max_ms": 69.949, "p95_ms": 61.18}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.701, "count": 107, "max_ms": 89.411, "p95_ms": 83.873}, "yolo11_640_fire_11class": {"avg_ms": 52.582, "count": 107, "max_ms": 70.488, "p95_ms": 67.987}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.835, "count": 107, "max_ms": 92.026, "p95_ms": 88.445}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.874, "count": 108, "max_ms": 295.498, "p95_ms": 254.638}, "read_interval": {"avg_ms": 240.501, "count": 107, "max_ms": 304.741, "p95_ms": 264.431}, "triton_wait": {"avg_ms": 66.944, "count": 107, "max_ms": 92.037, "p95_ms": 88.456}, "visualize": {"avg_ms": 10.827, "count": 107, "max_ms": 11.919, "p95_ms": 11.343}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3972.281, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5599a9cd1cc0] error while decoding MB 70 38, bytestream -20", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:01:09+09:00", "last_error_summary": "[h264 @ 0x5599a9cd1cc0] error while decoding MB 70 38, bytestream -20", "last_exit_at": "2026-07-01T16:59:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.789, "last_started_at": "2026-07-01T16:59:20+09:00", "pid": 10658, "process": {"exists": false, "pid": 10658}, "restart_count": 65, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.616, "sample_count": 1183, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34748, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:01:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T17:01:09+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T17:01:09+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T17:01:09.112506+09:00", "mtime_epoch": 1782892869.1125057, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178724}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:01:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.836, "name": "celery", "pid": 237, "rss_mb": 874.965, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.142, "pushed": 4.142, "read": 4.142}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.287, "count": 123, "max_ms": 13.195, "p95_ms": 10.633}, "gstreamer_push": {"avg_ms": 0.961, "count": 124, "max_ms": 3.086, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.212, "count": 123, "max_ms": 69.421, "p95_ms": 59.118}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.147, "count": 123, "max_ms": 89.287, "p95_ms": 79.66}, "yolo11_640_fire_11class": {"avg_ms": 52.575, "count": 123, "max_ms": 74.918, "p95_ms": 65.997}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.816, "count": 123, "max_ms": 91.237, "p95_ms": 81.317}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.247, "count": 124, "max_ms": 303.488, "p95_ms": 256.531}, "read_interval": {"avg_ms": 242.812, "count": 124, "max_ms": 313.58, "p95_ms": 265.628}, "triton_wait": {"avg_ms": 66.044, "count": 123, "max_ms": 91.251, "p95_ms": 83.277}, "visualize": {"avg_ms": 10.931, "count": 123, "max_ms": 12.525, "p95_ms": 11.735}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3971.094, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:01:09+09:00", "last_error_summary": "[h264 @ 0x5599a9cd1cc0] error while decoding MB 70 38, bytestream -20", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:01:09+09:00", "last_error_summary": "[h264 @ 0x5599a9cd1cc0] error while decoding MB 70 38, bytestream -20", "last_exit_at": "2026-07-01T17:01:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.989, "last_started_at": "2026-07-01T16:59:20+09:00", "pid": 10658, "process": {"exists": false, "pid": 10658}, "restart_count": 65, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.616, "sample_count": 1184, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34749, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:01:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:01:09+09:00", "last_frame_read_age_sec": 0.252, "last_frame_read_at": "2026-07-01T17:01:09+09:00", "mjpeg": {"age_sec": 0.246, "exists": true, "mtime": "2026-07-01T17:01:09.112506+09:00", "mtime_epoch": 1782892869.1125057, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178724}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:01:09+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.836, "name": "celery", "pid": 237, "rss_mb": 874.965, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.15, "pushed": 4.15, "read": 4.116}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.278, "count": 123, "max_ms": 13.195, "p95_ms": 10.633}, "gstreamer_push": {"avg_ms": 0.959, "count": 124, "max_ms": 3.086, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.117, "count": 123, "max_ms": 69.421, "p95_ms": 59.083}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.103, "count": 123, "max_ms": 89.287, "p95_ms": 79.66}, "yolo11_640_fire_11class": {"avg_ms": 52.526, "count": 123, "max_ms": 74.918, "p95_ms": 65.318}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.725, "count": 123, "max_ms": 91.237, "p95_ms": 80.808}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.463, "count": 124, "max_ms": 303.488, "p95_ms": 256.531}, "read_interval": {"avg_ms": 242.91, "count": 123, "max_ms": 313.58, "p95_ms": 265.628}, "triton_wait": {"avg_ms": 65.953, "count": 123, "max_ms": 91.251, "p95_ms": 83.277}, "visualize": {"avg_ms": 10.932, "count": 123, "max_ms": 12.525, "p95_ms": 11.735}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3971.332, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:01:09+09:00", "last_error_summary": "[h264 @ 0x5599a9cd1cc0] error while decoding MB 70 38, bytestream -20", "last_exit_at": "2026-07-01T17:01:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.994, "last_started_at": "2026-07-01T16:59:20+09:00", "pid": 10658, "process": {"exists": false, "pid": 10658}, "restart_count": 65, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.616, "sample_count": 1184, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34749, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:01:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T17:01:09+09:00", "last_frame_read_age_sec": 0.257, "last_frame_read_at": "2026-07-01T17:01:09+09:00", "mjpeg": {"age_sec": 0.251, "exists": true, "mtime": "2026-07-01T17:01:09.112506+09:00", "mtime_epoch": 1782892869.1125057, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178724}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:01:09+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.836, "name": "celery", "pid": 237, "rss_mb": 874.965, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.149, "pushed": 4.149, "read": 4.116}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.278, "count": 123, "max_ms": 13.195, "p95_ms": 10.633}, "gstreamer_push": {"avg_ms": 0.959, "count": 124, "max_ms": 3.086, "p95_ms": 1.455}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.117, "count": 123, "max_ms": 69.421, "p95_ms": 59.083}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.103, "count": 123, "max_ms": 89.287, "p95_ms": 79.66}, "yolo11_640_fire_11class": {"avg_ms": 52.526, "count": 123, "max_ms": 74.918, "p95_ms": 65.318}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.725, "count": 123, "max_ms": 91.237, "p95_ms": 80.808}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.463, "count": 124, "max_ms": 303.488, "p95_ms": 256.531}, "read_interval": {"avg_ms": 242.91, "count": 123, "max_ms": 313.58, "p95_ms": 265.628}, "triton_wait": {"avg_ms": 65.953, "count": 123, "max_ms": 91.251, "p95_ms": 83.277}, "visualize": {"avg_ms": 10.932, "count": 123, "max_ms": 12.525, "p95_ms": 11.735}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 289}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4081.695, "limit_mb": 8796093022207.996}, "detail": {"pid": 10765, "restart_count": 66, "start_count": 67}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:01:09+09:00", "last_error_summary": "[h264 @ 0x5599a9cd1cc0] error while decoding MB 70 38, bytestream -20", "last_exit_at": "2026-07-01T17:01:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:01:12+09:00", "pid": 10765, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.543, "name": "ffmpeg", "pid": 10765, "rss_mb": 136.543, "state": "S (sleeping)", "threads": 106}, "restart_count": 66, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 269.682, "sample_count": 1165, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 34749, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:01:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.961, "last_frame_pushed_at": "2026-07-01T17:01:09+09:00", "last_frame_read_age_sec": 5.21, "last_frame_read_at": "2026-07-01T17:01:09+09:00", "mjpeg": {"age_sec": 4.948, "exists": true, "mtime": "2026-07-01T17:01:09.368511+09:00", "mtime_epoch": 1782892869.368511, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174974}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:01:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.836, "name": "celery", "pid": 237, "rss_mb": 874.965, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.457, "pushed": 3.457, "read": 3.423}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.259, "count": 103, "max_ms": 13.195, "p95_ms": 10.633}, "gstreamer_push": {"avg_ms": 0.951, "count": 103, "max_ms": 3.086, "p95_ms": 1.374}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.634, "count": 103, "max_ms": 69.421, "p95_ms": 59.118}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.235, "count": 103, "max_ms": 89.287, "p95_ms": 79.66}, "yolo11_640_fire_11class": {"avg_ms": 52.946, "count": 103, "max_ms": 74.918, "p95_ms": 66.291}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.065, "count": 103, "max_ms": 91.237, "p95_ms": 80.808}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.919, "count": 103, "max_ms": 303.488, "p95_ms": 259.033}, "read_interval": {"avg_ms": 243.296, "count": 102, "max_ms": 313.58, "p95_ms": 268.187}, "triton_wait": {"avg_ms": 66.325, "count": 103, "max_ms": 91.251, "p95_ms": 84.405}, "visualize": {"avg_ms": 10.93, "count": 103, "max_ms": 12.525, "p95_ms": 11.818}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.797, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 293}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3972.816, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5650b161e700] error while decoding MB 25 33, bytestream -26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:03:01+09:00", "last_error_summary": "[h264 @ 0x5650b161e700] error while decoding MB 25 33, bytestream -26", "last_exit_at": "2026-07-01T17:01:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.457, "last_started_at": "2026-07-01T17:01:12+09:00", "pid": 10765, "process": {"exists": false, "pid": 10765}, "restart_count": 66, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 269.697, "sample_count": 1184, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35188, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:03:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T17:03:01+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-01T17:03:01+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-01T17:03:01.734799+09:00", "mtime_epoch": 1782892981.7347991, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180025}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:03:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 881.035, "name": "celery", "pid": 237, "rss_mb": 875.973, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.106, "pushed": 4.106, "read": 4.106}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.618, "count": 122, "max_ms": 17.746, "p95_ms": 11.975}, "gstreamer_push": {"avg_ms": 0.966, "count": 123, "max_ms": 2.993, "p95_ms": 1.818}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.26, "count": 122, "max_ms": 71.378, "p95_ms": 60.494}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.557, "count": 122, "max_ms": 89.131, "p95_ms": 81.756}, "yolo11_640_fire_11class": {"avg_ms": 52.851, "count": 122, "max_ms": 71.897, "p95_ms": 66.593}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.433, "count": 122, "max_ms": 92.27, "p95_ms": 86.688}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.791, "count": 123, "max_ms": 312.21, "p95_ms": 278.556}, "read_interval": {"avg_ms": 245.169, "count": 123, "max_ms": 322.803, "p95_ms": 285.037}, "triton_wait": {"avg_ms": 66.583, "count": 122, "max_ms": 92.283, "p95_ms": 86.704}, "visualize": {"avg_ms": 11.164, "count": 122, "max_ms": 14.726, "p95_ms": 13.085}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 293}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3972.082, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:03:01+09:00", "last_error_summary": "[h264 @ 0x5650b161e700] error while decoding MB 25 33, bytestream -26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:03:01+09:00", "last_error_summary": "[h264 @ 0x5650b161e700] error while decoding MB 25 33, bytestream -26", "last_exit_at": "2026-07-01T17:03:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.631, "last_started_at": "2026-07-01T17:01:12+09:00", "pid": 10765, "process": {"exists": false, "pid": 10765}, "restart_count": 66, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 269.697, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35189, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:03:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:03:01+09:00", "last_frame_read_age_sec": 0.222, "last_frame_read_at": "2026-07-01T17:03:01+09:00", "mjpeg": {"age_sec": 0.212, "exists": true, "mtime": "2026-07-01T17:03:01.734799+09:00", "mtime_epoch": 1782892981.7347991, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180025}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:03:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 881.035, "name": "celery", "pid": 237, "rss_mb": 875.973, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.113, "pushed": 4.113, "read": 4.08}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.62, "count": 122, "max_ms": 17.746, "p95_ms": 11.975}, "gstreamer_push": {"avg_ms": 0.967, "count": 123, "max_ms": 2.993, "p95_ms": 1.818}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.239, "count": 122, "max_ms": 71.378, "p95_ms": 60.494}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.52, "count": 122, "max_ms": 89.131, "p95_ms": 81.756}, "yolo11_640_fire_11class": {"avg_ms": 52.808, "count": 122, "max_ms": 71.897, "p95_ms": 66.593}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.399, "count": 122, "max_ms": 92.27, "p95_ms": 86.688}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.573, "count": 123, "max_ms": 312.21, "p95_ms": 278.556}, "read_interval": {"avg_ms": 245.097, "count": 122, "max_ms": 322.803, "p95_ms": 285.037}, "triton_wait": {"avg_ms": 66.549, "count": 122, "max_ms": 92.283, "p95_ms": 86.704}, "visualize": {"avg_ms": 11.163, "count": 122, "max_ms": 14.726, "p95_ms": 13.085}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 293}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3972.074, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:03:01+09:00", "last_error_summary": "[h264 @ 0x5650b161e700] error while decoding MB 25 33, bytestream -26", "last_exit_at": "2026-07-01T17:03:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.635, "last_started_at": "2026-07-01T17:01:12+09:00", "pid": 10765, "process": {"exists": false, "pid": 10765}, "restart_count": 66, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 269.697, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35189, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:03:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:03:01+09:00", "last_frame_read_age_sec": 0.225, "last_frame_read_at": "2026-07-01T17:03:01+09:00", "mjpeg": {"age_sec": 0.215, "exists": true, "mtime": "2026-07-01T17:03:01.734799+09:00", "mtime_epoch": 1782892981.7347991, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180025}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:03:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 881.035, "name": "celery", "pid": 237, "rss_mb": 875.973, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.113, "pushed": 4.113, "read": 4.079}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.62, "count": 122, "max_ms": 17.746, "p95_ms": 11.975}, "gstreamer_push": {"avg_ms": 0.967, "count": 123, "max_ms": 2.993, "p95_ms": 1.818}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.239, "count": 122, "max_ms": 71.378, "p95_ms": 60.494}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.52, "count": 122, "max_ms": 89.131, "p95_ms": 81.756}, "yolo11_640_fire_11class": {"avg_ms": 52.808, "count": 122, "max_ms": 71.897, "p95_ms": 66.593}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.399, "count": 122, "max_ms": 92.27, "p95_ms": 86.688}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.573, "count": 123, "max_ms": 312.21, "p95_ms": 278.556}, "read_interval": {"avg_ms": 245.097, "count": 122, "max_ms": 322.803, "p95_ms": 285.037}, "triton_wait": {"avg_ms": 66.549, "count": 122, "max_ms": 92.283, "p95_ms": 86.704}, "visualize": {"avg_ms": 11.163, "count": 122, "max_ms": 14.726, "p95_ms": 13.085}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 293}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4077.367, "limit_mb": 8796093022207.996}, "detail": {"pid": 10872, "restart_count": 67, "start_count": 68}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:03:01+09:00", "last_error_summary": "[h264 @ 0x5650b161e700] error while decoding MB 25 33, bytestream -26", "last_exit_at": "2026-07-01T17:03:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:03:04+09:00", "pid": 10872, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.922, "name": "ffmpeg", "pid": 10872, "rss_mb": 136.922, "state": "S (sleeping)", "threads": 106}, "restart_count": 67, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 269.682, "sample_count": 1169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35189, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:03:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.117, "last_frame_pushed_at": "2026-07-01T17:03:01+09:00", "last_frame_read_age_sec": 4.337, "last_frame_read_at": "2026-07-01T17:03:01+09:00", "mjpeg": {"age_sec": 4.099, "exists": true, "mtime": "2026-07-01T17:03:01.962804+09:00", "mtime_epoch": 1782892981.9628038, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180129}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:03:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 881.035, "name": "celery", "pid": 237, "rss_mb": 875.973, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.552, "pushed": 3.552, "read": 3.518}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.721, "count": 105, "max_ms": 17.746, "p95_ms": 11.975}, "gstreamer_push": {"avg_ms": 0.955, "count": 106, "max_ms": 2.957, "p95_ms": 1.818}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.332, "count": 105, "max_ms": 71.378, "p95_ms": 60.494}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.609, "count": 105, "max_ms": 85.685, "p95_ms": 81.379}, "yolo11_640_fire_11class": {"avg_ms": 52.91, "count": 105, "max_ms": 71.897, "p95_ms": 65.992}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.661, "count": 105, "max_ms": 92.27, "p95_ms": 86.687}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.597, "count": 106, "max_ms": 312.21, "p95_ms": 278.556}, "read_interval": {"avg_ms": 245.915, "count": 105, "max_ms": 322.803, "p95_ms": 285.037}, "triton_wait": {"avg_ms": 66.785, "count": 105, "max_ms": 92.283, "p95_ms": 86.7}, "visualize": {"avg_ms": 11.196, "count": 105, "max_ms": 14.726, "p95_ms": 13.085}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 296}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3976.91, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ba53cdf580] error while decoding MB 40 34, bytestream -26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:04:53+09:00", "last_error_summary": "[h264 @ 0x55ba53cdf580] error while decoding MB 40 34, bytestream -26", "last_exit_at": "2026-07-01T17:03:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.566, "last_started_at": "2026-07-01T17:03:04+09:00", "pid": 10872, "process": {"exists": false, "pid": 10872}, "restart_count": 67, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.572, "sample_count": 1182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35621, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:04:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T17:04:53+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T17:04:53+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T17:04:53.589054+09:00", "mtime_epoch": 1782893093.5890543, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177310}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:04:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 881.16, "name": "celery", "pid": 237, "rss_mb": 878.051, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.012, "pushed": 4.012, "read": 4.012}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.725, "count": 119, "max_ms": 16.925, "p95_ms": 10.79}, "gstreamer_push": {"avg_ms": 0.924, "count": 120, "max_ms": 2.883, "p95_ms": 0.994}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.132, "count": 119, "max_ms": 63.423, "p95_ms": 57.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.737, "count": 119, "max_ms": 90.754, "p95_ms": 85.771}, "yolo11_640_fire_11class": {"avg_ms": 52.096, "count": 119, "max_ms": 70.786, "p95_ms": 68.585}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.038, "count": 119, "max_ms": 89.892, "p95_ms": 86.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.183, "count": 120, "max_ms": 320.194, "p95_ms": 272.758}, "read_interval": {"avg_ms": 250.79, "count": 120, "max_ms": 331.315, "p95_ms": 285.676}, "triton_wait": {"avg_ms": 66.372, "count": 119, "max_ms": 90.767, "p95_ms": 86.957}, "visualize": {"avg_ms": 11.433, "count": 119, "max_ms": 17.692, "p95_ms": 13.403}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 296}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3976.035, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:04:53+09:00", "last_error_summary": "[h264 @ 0x55ba53cdf580] error while decoding MB 40 34, bytestream -26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:04:53+09:00", "last_error_summary": "[h264 @ 0x55ba53cdf580] error while decoding MB 40 34, bytestream -26", "last_exit_at": "2026-07-01T17:04:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.745, "last_started_at": "2026-07-01T17:03:04+09:00", "pid": 10872, "process": {"exists": false, "pid": 10872}, "restart_count": 67, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.876, "sample_count": 1182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35622, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:04:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T17:04:53+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T17:04:53+09:00", "mjpeg": {"age_sec": 0.216, "exists": true, "mtime": "2026-07-01T17:04:53.589054+09:00", "mtime_epoch": 1782893093.5890543, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177310}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:04:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 881.16, "name": "celery", "pid": 237, "rss_mb": 878.051, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.021, "pushed": 4.021, "read": 3.987}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.722, "count": 120, "max_ms": 16.925, "p95_ms": 10.683}, "gstreamer_push": {"avg_ms": 0.923, "count": 120, "max_ms": 2.883, "p95_ms": 0.994}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.066, "count": 120, "max_ms": 63.423, "p95_ms": 56.727}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.709, "count": 120, "max_ms": 90.754, "p95_ms": 83.571}, "yolo11_640_fire_11class": {"avg_ms": 52.043, "count": 120, "max_ms": 70.786, "p95_ms": 68.329}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.0, "count": 120, "max_ms": 89.892, "p95_ms": 85.776}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.059, "count": 120, "max_ms": 320.194, "p95_ms": 272.758}, "read_interval": {"avg_ms": 250.797, "count": 119, "max_ms": 331.315, "p95_ms": 293.668}, "triton_wait": {"avg_ms": 66.332, "count": 120, "max_ms": 90.767, "p95_ms": 85.792}, "visualize": {"avg_ms": 11.427, "count": 120, "max_ms": 17.692, "p95_ms": 13.344}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 296}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3976.152, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:04:53+09:00", "last_error_summary": "[h264 @ 0x55ba53cdf580] error while decoding MB 40 34, bytestream -26", "last_exit_at": "2026-07-01T17:04:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.748, "last_started_at": "2026-07-01T17:03:04+09:00", "pid": 10872, "process": {"exists": false, "pid": 10872}, "restart_count": 67, "restart_count_per_hour": 30, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.876, "sample_count": 1182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35622, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:04:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T17:04:53+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-01T17:04:53+09:00", "mjpeg": {"age_sec": 0.219, "exists": true, "mtime": "2026-07-01T17:04:53.589054+09:00", "mtime_epoch": 1782893093.5890543, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177310}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:04:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 881.16, "name": "celery", "pid": 237, "rss_mb": 878.051, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.02, "pushed": 4.02, "read": 3.987}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.722, "count": 120, "max_ms": 16.925, "p95_ms": 10.683}, "gstreamer_push": {"avg_ms": 0.923, "count": 120, "max_ms": 2.883, "p95_ms": 0.994}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.066, "count": 120, "max_ms": 63.423, "p95_ms": 56.727}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.709, "count": 120, "max_ms": 90.754, "p95_ms": 83.571}, "yolo11_640_fire_11class": {"avg_ms": 52.043, "count": 120, "max_ms": 70.786, "p95_ms": 68.329}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.0, "count": 120, "max_ms": 89.892, "p95_ms": 85.776}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.059, "count": 120, "max_ms": 320.194, "p95_ms": 272.758}, "read_interval": {"avg_ms": 250.797, "count": 119, "max_ms": 331.315, "p95_ms": 293.668}, "triton_wait": {"avg_ms": 66.332, "count": 120, "max_ms": 90.767, "p95_ms": 85.792}, "visualize": {"avg_ms": 11.427, "count": 120, "max_ms": 17.692, "p95_ms": 13.344}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.849, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 296}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4083.398, "limit_mb": 8796093022207.996}, "detail": {"pid": 10979, "restart_count": 68, "start_count": 69}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:04:53+09:00", "last_error_summary": "[h264 @ 0x55ba53cdf580] error while decoding MB 40 34, bytestream -26", "last_exit_at": "2026-07-01T17:04:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:04:56+09:00", "pid": 10979, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.121, "name": "ffmpeg", "pid": 10979, "rss_mb": 139.121, "state": "S (sleeping)", "threads": 106}, "restart_count": 68, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.876, "sample_count": 1162, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 35622, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:04:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.929, "last_frame_pushed_at": "2026-07-01T17:04:53+09:00", "last_frame_read_age_sec": 5.155, "last_frame_read_at": "2026-07-01T17:04:53+09:00", "mjpeg": {"age_sec": 4.913, "exists": true, "mtime": "2026-07-01T17:04:53.817059+09:00", "mtime_epoch": 1782893093.817059, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177404}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:04:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 881.16, "name": "celery", "pid": 237, "rss_mb": 878.051, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.336, "pushed": 3.336, "read": 3.302}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.719, "count": 99, "max_ms": 16.925, "p95_ms": 10.623}, "gstreamer_push": {"avg_ms": 0.926, "count": 100, "max_ms": 2.883, "p95_ms": 0.993}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.408, "count": 99, "max_ms": 63.423, "p95_ms": 58.199}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.733, "count": 99, "max_ms": 88.857, "p95_ms": 85.771}, "yolo11_640_fire_11class": {"avg_ms": 52.061, "count": 99, "max_ms": 70.786, "p95_ms": 68.329}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.227, "count": 99, "max_ms": 89.892, "p95_ms": 86.45}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.51, "count": 100, "max_ms": 320.194, "p95_ms": 284.614}, "read_interval": {"avg_ms": 253.316, "count": 99, "max_ms": 331.315, "p95_ms": 296.369}, "triton_wait": {"avg_ms": 66.477, "count": 99, "max_ms": 89.908, "p95_ms": 86.957}, "visualize": {"avg_ms": 11.539, "count": 99, "max_ms": 17.692, "p95_ms": 13.414}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 300}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3979.777, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:04:53+09:00", "last_error_summary": "[h264 @ 0x55ba53cdf580] error while decoding MB 40 34, bytestream -26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:04:53+09:00", "last_error_summary": "[h264 @ 0x55ba53cdf580] error while decoding MB 40 34, bytestream -26", "last_exit_at": "2026-07-01T17:06:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 100.534, "last_started_at": "2026-07-01T17:04:56+09:00", "pid": 10979, "process": {"exists": false, "pid": 10979}, "restart_count": 68, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.758, "sample_count": 1180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36021, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:06:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:06:37+09:00", "last_frame_read_age_sec": 0.257, "last_frame_read_at": "2026-07-01T17:06:37+09:00", "mjpeg": {"age_sec": 0.248, "exists": true, "mtime": "2026-07-01T17:06:37.015123+09:00", "mtime_epoch": 1782893197.0151234, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174099}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:06:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 881.898, "name": "celery", "pid": 237, "rss_mb": 881.418, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.002, "pushed": 4.002, "read": 3.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.646, "count": 119, "max_ms": 10.928, "p95_ms": 10.648}, "gstreamer_push": {"avg_ms": 0.943, "count": 120, "max_ms": 2.936, "p95_ms": 1.794}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.507, "count": 119, "max_ms": 70.431, "p95_ms": 62.934}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.461, "count": 119, "max_ms": 91.211, "p95_ms": 82.968}, "yolo11_640_fire_11class": {"avg_ms": 53.044, "count": 119, "max_ms": 73.887, "p95_ms": 68.102}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.06, "count": 119, "max_ms": 93.205, "p95_ms": 87.088}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.186, "count": 120, "max_ms": 331.826, "p95_ms": 277.385}, "read_interval": {"avg_ms": 251.746, "count": 119, "max_ms": 341.854, "p95_ms": 293.793}, "triton_wait": {"avg_ms": 67.333, "count": 119, "max_ms": 93.218, "p95_ms": 87.664}, "visualize": {"avg_ms": 11.376, "count": 119, "max_ms": 15.066, "p95_ms": 12.453}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 300}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3979.77, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:04:53+09:00", "last_error_summary": "[h264 @ 0x55ba53cdf580] error while decoding MB 40 34, bytestream -26", "last_exit_at": "2026-07-01T17:06:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 100.537, "last_started_at": "2026-07-01T17:04:56+09:00", "pid": 10979, "process": {"exists": false, "pid": 10979}, "restart_count": 68, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.758, "sample_count": 1180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36021, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:06:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T17:06:37+09:00", "last_frame_read_age_sec": 0.26, "last_frame_read_at": "2026-07-01T17:06:37+09:00", "mjpeg": {"age_sec": 0.251, "exists": true, "mtime": "2026-07-01T17:06:37.015123+09:00", "mtime_epoch": 1782893197.0151234, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174099}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:06:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 881.898, "name": "celery", "pid": 237, "rss_mb": 881.418, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.001, "pushed": 4.001, "read": 3.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.646, "count": 119, "max_ms": 10.928, "p95_ms": 10.648}, "gstreamer_push": {"avg_ms": 0.943, "count": 120, "max_ms": 2.936, "p95_ms": 1.794}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.507, "count": 119, "max_ms": 70.431, "p95_ms": 62.934}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.461, "count": 119, "max_ms": 91.211, "p95_ms": 82.968}, "yolo11_640_fire_11class": {"avg_ms": 53.044, "count": 119, "max_ms": 73.887, "p95_ms": 68.102}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.06, "count": 119, "max_ms": 93.205, "p95_ms": 87.088}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.186, "count": 120, "max_ms": 331.826, "p95_ms": 277.385}, "read_interval": {"avg_ms": 251.746, "count": 119, "max_ms": 341.854, "p95_ms": 293.793}, "triton_wait": {"avg_ms": 67.333, "count": 119, "max_ms": 93.218, "p95_ms": 87.664}, "visualize": {"avg_ms": 11.376, "count": 119, "max_ms": 15.066, "p95_ms": 12.453}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 300}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4087.918, "limit_mb": 8796093022207.996}, "detail": {"pid": 11086, "restart_count": 69, "start_count": 70}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:04:53+09:00", "last_error_summary": "[h264 @ 0x55ba53cdf580] error while decoding MB 40 34, bytestream -26", "last_exit_at": "2026-07-01T17:06:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:06:39+09:00", "pid": 11086, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.488, "name": "ffmpeg", "pid": 11086, "rss_mb": 139.488, "state": "S (sleeping)", "threads": 106}, "restart_count": 69, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.761, "sample_count": 1163, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36021, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:06:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.148, "last_frame_pushed_at": "2026-07-01T17:06:37+09:00", "last_frame_read_age_sec": 4.402, "last_frame_read_at": "2026-07-01T17:06:37+09:00", "mjpeg": {"age_sec": 4.132, "exists": true, "mtime": "2026-07-01T17:06:37.275129+09:00", "mtime_epoch": 1782893197.2751286, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169231}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:06:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 881.898, "name": "celery", "pid": 237, "rss_mb": 881.418, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.438, "pushed": 3.438, "read": 3.405}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.701, "count": 102, "max_ms": 10.928, "p95_ms": 10.648}, "gstreamer_push": {"avg_ms": 0.956, "count": 103, "max_ms": 2.936, "p95_ms": 1.803}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.974, "count": 102, "max_ms": 70.431, "p95_ms": 62.934}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.86, "count": 102, "max_ms": 91.211, "p95_ms": 82.968}, "yolo11_640_fire_11class": {"avg_ms": 53.552, "count": 102, "max_ms": 73.887, "p95_ms": 68.102}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.632, "count": 102, "max_ms": 93.205, "p95_ms": 87.088}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.296, "count": 103, "max_ms": 331.826, "p95_ms": 284.633}, "read_interval": {"avg_ms": 252.883, "count": 102, "max_ms": 341.854, "p95_ms": 293.793}, "triton_wait": {"avg_ms": 67.843, "count": 102, "max_ms": 93.218, "p95_ms": 87.664}, "visualize": {"avg_ms": 11.397, "count": 102, "max_ms": 15.066, "p95_ms": 12.453}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 304}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3984.145, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5620cc1f1140] error while decoding MB 35 24, bytestream -38", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:08:35+09:00", "last_error_summary": "[h264 @ 0x5620cc1f1140] error while decoding MB 35 24, bytestream -38", "last_exit_at": "2026-07-01T17:06:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.99, "last_started_at": "2026-07-01T17:06:39+09:00", "pid": 11086, "process": {"exists": false, "pid": 11086}, "restart_count": 69, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.588, "sample_count": 1170, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36480, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:08:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T17:08:35+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T17:08:35+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T17:08:35.349475+09:00", "mtime_epoch": 1782893315.3494751, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181664}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:08:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 881.602, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.054, "pushed": 4.054, "read": 4.054}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.488, "count": 121, "max_ms": 10.944, "p95_ms": 10.265}, "gstreamer_push": {"avg_ms": 0.893, "count": 121, "max_ms": 1.86, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.477, "count": 121, "max_ms": 63.709, "p95_ms": 61.236}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.551, "count": 121, "max_ms": 86.794, "p95_ms": 83.165}, "yolo11_640_fire_11class": {"avg_ms": 53.728, "count": 121, "max_ms": 72.413, "p95_ms": 71.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.747, "count": 121, "max_ms": 91.562, "p95_ms": 88.727}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.479, "count": 121, "max_ms": 295.663, "p95_ms": 270.781}, "read_interval": {"avg_ms": 248.135, "count": 121, "max_ms": 305.784, "p95_ms": 280.647}, "triton_wait": {"avg_ms": 68.381, "count": 121, "max_ms": 91.574, "p95_ms": 88.74}, "visualize": {"avg_ms": 11.215, "count": 121, "max_ms": 13.222, "p95_ms": 12.209}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.848, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 304}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3983.262, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:08:35+09:00", "last_error_summary": "[h264 @ 0x5620cc1f1140] error while decoding MB 35 24, bytestream -38", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:08:35+09:00", "last_error_summary": "[h264 @ 0x5620cc1f1140] error while decoding MB 35 24, bytestream -38", "last_exit_at": "2026-07-01T17:08:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 116.17, "last_started_at": "2026-07-01T17:06:39+09:00", "pid": 11086, "process": {"exists": false, "pid": 11086}, "restart_count": 69, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.588, "sample_count": 1171, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36481, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:08:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T17:08:35+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T17:08:35+09:00", "mjpeg": {"age_sec": 0.227, "exists": true, "mtime": "2026-07-01T17:08:35.349475+09:00", "mtime_epoch": 1782893315.3494751, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181664}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:08:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 881.602, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.068, "pushed": 4.068, "read": 4.034}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.484, "count": 121, "max_ms": 10.944, "p95_ms": 10.265}, "gstreamer_push": {"avg_ms": 0.894, "count": 121, "max_ms": 1.86, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.411, "count": 121, "max_ms": 63.709, "p95_ms": 61.236}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.552, "count": 121, "max_ms": 86.794, "p95_ms": 83.165}, "yolo11_640_fire_11class": {"avg_ms": 53.654, "count": 121, "max_ms": 72.413, "p95_ms": 71.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.711, "count": 121, "max_ms": 91.562, "p95_ms": 88.727}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.454, "count": 121, "max_ms": 295.663, "p95_ms": 270.781}, "read_interval": {"avg_ms": 248.196, "count": 120, "max_ms": 305.784, "p95_ms": 280.647}, "triton_wait": {"avg_ms": 68.364, "count": 121, "max_ms": 91.574, "p95_ms": 88.74}, "visualize": {"avg_ms": 11.216, "count": 121, "max_ms": 13.222, "p95_ms": 12.209}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.746, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 304}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3983.16, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:08:35+09:00", "last_error_summary": "[h264 @ 0x5620cc1f1140] error while decoding MB 35 24, bytestream -38", "last_exit_at": "2026-07-01T17:08:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 116.173, "last_started_at": "2026-07-01T17:06:39+09:00", "pid": 11086, "process": {"exists": false, "pid": 11086}, "restart_count": 69, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.588, "sample_count": 1171, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36481, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:08:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T17:08:35+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T17:08:35+09:00", "mjpeg": {"age_sec": 0.23, "exists": true, "mtime": "2026-07-01T17:08:35.349475+09:00", "mtime_epoch": 1782893315.3494751, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181664}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:08:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 881.602, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.067, "pushed": 4.067, "read": 4.034}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.484, "count": 121, "max_ms": 10.944, "p95_ms": 10.265}, "gstreamer_push": {"avg_ms": 0.894, "count": 121, "max_ms": 1.86, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.411, "count": 121, "max_ms": 63.709, "p95_ms": 61.236}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.552, "count": 121, "max_ms": 86.794, "p95_ms": 83.165}, "yolo11_640_fire_11class": {"avg_ms": 53.654, "count": 121, "max_ms": 72.413, "p95_ms": 71.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.711, "count": 121, "max_ms": 91.562, "p95_ms": 88.727}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.454, "count": 121, "max_ms": 295.663, "p95_ms": 270.781}, "read_interval": {"avg_ms": 248.196, "count": 120, "max_ms": 305.784, "p95_ms": 280.647}, "triton_wait": {"avg_ms": 68.364, "count": 121, "max_ms": 91.574, "p95_ms": 88.74}, "visualize": {"avg_ms": 11.216, "count": 121, "max_ms": 13.222, "p95_ms": 12.209}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.749, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 304}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4083.195, "limit_mb": 8796093022207.996}, "detail": {"pid": 11193, "restart_count": 70, "start_count": 71}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:08:35+09:00", "last_error_summary": "[h264 @ 0x5620cc1f1140] error while decoding MB 35 24, bytestream -38", "last_exit_at": "2026-07-01T17:08:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:08:38+09:00", "pid": 11193, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.02, "name": "ffmpeg", "pid": 11193, "rss_mb": 136.02, "state": "S (sleeping)", "threads": 106}, "restart_count": 70, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.22, "sample_count": 1152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36481, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:08:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.477, "last_frame_pushed_at": "2026-07-01T17:08:35+09:00", "last_frame_read_age_sec": 4.708, "last_frame_read_at": "2026-07-01T17:08:35+09:00", "mjpeg": {"age_sec": 4.46, "exists": true, "mtime": "2026-07-01T17:08:35.589480+09:00", "mtime_epoch": 1782893315.58948, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177734}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:08:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 881.602, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.486, "pushed": 3.486, "read": 3.452}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.466, "count": 104, "max_ms": 10.944, "p95_ms": 10.388}, "gstreamer_push": {"avg_ms": 0.897, "count": 104, "max_ms": 1.86, "p95_ms": 0.937}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.808, "count": 104, "max_ms": 63.709, "p95_ms": 61.215}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.846, "count": 104, "max_ms": 86.22, "p95_ms": 83.165}, "yolo11_640_fire_11class": {"avg_ms": 53.807, "count": 104, "max_ms": 72.413, "p95_ms": 71.003}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.989, "count": 104, "max_ms": 91.562, "p95_ms": 88.727}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.685, "count": 104, "max_ms": 295.663, "p95_ms": 269.856}, "read_interval": {"avg_ms": 246.378, "count": 103, "max_ms": 305.784, "p95_ms": 280.366}, "triton_wait": {"avg_ms": 68.644, "count": 104, "max_ms": 91.574, "p95_ms": 88.74}, "visualize": {"avg_ms": 11.102, "count": 104, "max_ms": 13.222, "p95_ms": 12.16}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.837, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 307}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3976.941, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562602d32a40] error while decoding MB 77 29, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:10:29+09:00", "last_error_summary": "[h264 @ 0x562602d32a40] error while decoding MB 77 29, bytestream -7", "last_exit_at": "2026-07-01T17:08:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.76, "last_started_at": "2026-07-01T17:08:38+09:00", "pid": 11193, "process": {"exists": false, "pid": 11193}, "restart_count": 70, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.877, "sample_count": 1158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36908, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:10:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T17:10:29+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T17:10:29+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T17:10:29.767737+09:00", "mtime_epoch": 1782893429.7677367, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181840}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:10:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 876.137, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.911, "pushed": 3.911, "read": 3.911}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.926, "count": 116, "max_ms": 18.991, "p95_ms": 10.791}, "gstreamer_push": {"avg_ms": 0.939, "count": 117, "max_ms": 2.97, "p95_ms": 1.751}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.851, "count": 116, "max_ms": 74.587, "p95_ms": 69.489}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.668, "count": 116, "max_ms": 95.592, "p95_ms": 87.011}, "yolo11_640_fire_11class": {"avg_ms": 54.865, "count": 116, "max_ms": 78.291, "p95_ms": 72.387}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.874, "count": 116, "max_ms": 97.536, "p95_ms": 90.929}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.651, "count": 117, "max_ms": 319.459, "p95_ms": 278.047}, "read_interval": {"avg_ms": 257.253, "count": 117, "max_ms": 329.121, "p95_ms": 289.057}, "triton_wait": {"avg_ms": 70.022, "count": 116, "max_ms": 97.549, "p95_ms": 90.943}, "visualize": {"avg_ms": 11.642, "count": 116, "max_ms": 15.423, "p95_ms": 14.288}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 307}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3976.031, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:10:29+09:00", "last_error_summary": "[h264 @ 0x562602d32a40] error while decoding MB 77 29, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:10:29+09:00", "last_error_summary": "[h264 @ 0x562602d32a40] error while decoding MB 77 29, bytestream -7", "last_exit_at": "2026-07-01T17:10:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.012, "last_started_at": "2026-07-01T17:08:38+09:00", "pid": 11193, "process": {"exists": false, "pid": 11193}, "restart_count": 70, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.042, "sample_count": 1158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36909, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:10:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:10:30+09:00", "last_frame_read_age_sec": 0.303, "last_frame_read_at": "2026-07-01T17:10:29+09:00", "mjpeg": {"age_sec": 0.293, "exists": true, "mtime": "2026-07-01T17:10:29.767737+09:00", "mtime_epoch": 1782893429.7677367, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181840}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:10:30+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 876.137, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.912, "pushed": 3.912, "read": 3.879}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.979, "count": 116, "max_ms": 18.991, "p95_ms": 12.486}, "gstreamer_push": {"avg_ms": 0.939, "count": 117, "max_ms": 2.97, "p95_ms": 1.751}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.773, "count": 116, "max_ms": 74.587, "p95_ms": 69.489}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.586, "count": 116, "max_ms": 95.592, "p95_ms": 87.011}, "yolo11_640_fire_11class": {"avg_ms": 54.784, "count": 116, "max_ms": 78.291, "p95_ms": 72.387}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.786, "count": 116, "max_ms": 97.536, "p95_ms": 90.929}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.184, "count": 117, "max_ms": 319.459, "p95_ms": 278.317}, "read_interval": {"avg_ms": 257.352, "count": 116, "max_ms": 329.121, "p95_ms": 289.057}, "triton_wait": {"avg_ms": 69.933, "count": 116, "max_ms": 97.549, "p95_ms": 90.943}, "visualize": {"avg_ms": 11.674, "count": 116, "max_ms": 15.592, "p95_ms": 14.455}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 307}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3976.023, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:10:29+09:00", "last_error_summary": "[h264 @ 0x562602d32a40] error while decoding MB 77 29, bytestream -7", "last_exit_at": "2026-07-01T17:10:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.015, "last_started_at": "2026-07-01T17:08:38+09:00", "pid": 11193, "process": {"exists": false, "pid": 11193}, "restart_count": 70, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.042, "sample_count": 1158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36909, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:10:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:10:30+09:00", "last_frame_read_age_sec": 0.306, "last_frame_read_at": "2026-07-01T17:10:29+09:00", "mjpeg": {"age_sec": 0.296, "exists": true, "mtime": "2026-07-01T17:10:29.767737+09:00", "mtime_epoch": 1782893429.7677367, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181840}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:10:30+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 876.137, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.912, "pushed": 3.912, "read": 3.878}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.979, "count": 116, "max_ms": 18.991, "p95_ms": 12.486}, "gstreamer_push": {"avg_ms": 0.939, "count": 117, "max_ms": 2.97, "p95_ms": 1.751}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.773, "count": 116, "max_ms": 74.587, "p95_ms": 69.489}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.586, "count": 116, "max_ms": 95.592, "p95_ms": 87.011}, "yolo11_640_fire_11class": {"avg_ms": 54.784, "count": 116, "max_ms": 78.291, "p95_ms": 72.387}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.786, "count": 116, "max_ms": 97.536, "p95_ms": 90.929}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.184, "count": 117, "max_ms": 319.459, "p95_ms": 278.317}, "read_interval": {"avg_ms": 257.352, "count": 116, "max_ms": 329.121, "p95_ms": 289.057}, "triton_wait": {"avg_ms": 69.933, "count": 116, "max_ms": 97.549, "p95_ms": 90.943}, "visualize": {"avg_ms": 11.674, "count": 116, "max_ms": 15.592, "p95_ms": 14.455}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 307}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4081.5, "limit_mb": 8796093022207.996}, "detail": {"pid": 11300, "restart_count": 71, "start_count": 72}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:10:29+09:00", "last_error_summary": "[h264 @ 0x562602d32a40] error while decoding MB 77 29, bytestream -7", "last_exit_at": "2026-07-01T17:10:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:10:33+09:00", "pid": 11300, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.949, "name": "ffmpeg", "pid": 11300, "rss_mb": 133.949, "state": "S (sleeping)", "threads": 106}, "restart_count": 71, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.232, "sample_count": 1138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 36909, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:10:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 5.012, "last_frame_pushed_at": "2026-07-01T17:10:30+09:00", "last_frame_read_age_sec": 5.312, "last_frame_read_at": "2026-07-01T17:10:29+09:00", "mjpeg": {"age_sec": 4.994, "exists": true, "mtime": "2026-07-01T17:10:30.075743+09:00", "mtime_epoch": 1782893430.0757427, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181121}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:10:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 876.137, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.251, "pushed": 3.251, "read": 3.218}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.971, "count": 97, "max_ms": 18.991, "p95_ms": 12.499}, "gstreamer_push": {"avg_ms": 0.943, "count": 97, "max_ms": 2.97, "p95_ms": 1.805}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.138, "count": 97, "max_ms": 74.587, "p95_ms": 70.338}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.165, "count": 97, "max_ms": 95.592, "p95_ms": 87.321}, "yolo11_640_fire_11class": {"avg_ms": 55.486, "count": 97, "max_ms": 78.291, "p95_ms": 73.688}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.765, "count": 97, "max_ms": 97.536, "p95_ms": 91.982}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.939, "count": 97, "max_ms": 319.459, "p95_ms": 278.317}, "read_interval": {"avg_ms": 258.031, "count": 96, "max_ms": 329.121, "p95_ms": 289.057}, "triton_wait": {"avg_ms": 70.86, "count": 97, "max_ms": 97.549, "p95_ms": 91.994}, "visualize": {"avg_ms": 11.69, "count": 97, "max_ms": 15.592, "p95_ms": 15.249}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.835, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 311}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3975.301, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d37dc15280] error while decoding MB 67 11, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:12:24+09:00", "last_error_summary": "[h264 @ 0x55d37dc15280] error while decoding MB 67 11, bytestream -5", "last_exit_at": "2026-07-01T17:10:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.166, "last_started_at": "2026-07-01T17:10:33+09:00", "pid": 11300, "process": {"exists": false, "pid": 11300}, "restart_count": 71, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.358, "sample_count": 1141, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37335, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:12:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T17:12:24+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T17:12:24+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T17:12:24.193001+09:00", "mtime_epoch": 1782893544.193001, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170138}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:12:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 874.234, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.923, "pushed": 3.923, "read": 3.923}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.001, "count": 117, "max_ms": 18.012, "p95_ms": 11.531}, "gstreamer_push": {"avg_ms": 0.886, "count": 117, "max_ms": 1.828, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.683, "count": 117, "max_ms": 69.432, "p95_ms": 59.831}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.369, "count": 117, "max_ms": 89.881, "p95_ms": 88.564}, "yolo11_640_fire_11class": {"avg_ms": 53.622, "count": 117, "max_ms": 76.238, "p95_ms": 71.337}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.562, "count": 117, "max_ms": 96.404, "p95_ms": 88.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.428, "count": 117, "max_ms": 307.938, "p95_ms": 271.206}, "read_interval": {"avg_ms": 256.541, "count": 117, "max_ms": 470.911, "p95_ms": 284.41}, "triton_wait": {"avg_ms": 67.954, "count": 117, "max_ms": 96.417, "p95_ms": 89.548}, "visualize": {"avg_ms": 11.68, "count": 117, "max_ms": 14.821, "p95_ms": 13.352}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.821, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 311}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3974.73, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:12:24+09:00", "last_error_summary": "[h264 @ 0x55d37dc15280] error while decoding MB 67 11, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:12:24+09:00", "last_error_summary": "[h264 @ 0x55d37dc15280] error while decoding MB 67 11, bytestream -5", "last_exit_at": "2026-07-01T17:12:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.344, "last_started_at": "2026-07-01T17:10:33+09:00", "pid": 11300, "process": {"exists": false, "pid": 11300}, "restart_count": 71, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.358, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:12:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:12:24+09:00", "last_frame_read_age_sec": 0.229, "last_frame_read_at": "2026-07-01T17:12:24+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-01T17:12:24.193001+09:00", "mtime_epoch": 1782893544.193001, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170138}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:12:24+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 874.234, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.934, "pushed": 3.934, "read": 3.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.004, "count": 117, "max_ms": 18.012, "p95_ms": 11.531}, "gstreamer_push": {"avg_ms": 0.886, "count": 118, "max_ms": 1.828, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.637, "count": 117, "max_ms": 69.432, "p95_ms": 59.831}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.348, "count": 117, "max_ms": 89.881, "p95_ms": 88.564}, "yolo11_640_fire_11class": {"avg_ms": 53.595, "count": 117, "max_ms": 76.238, "p95_ms": 71.337}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.534, "count": 117, "max_ms": 96.404, "p95_ms": 88.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.255, "count": 118, "max_ms": 307.938, "p95_ms": 271.206}, "read_interval": {"avg_ms": 256.541, "count": 117, "max_ms": 470.911, "p95_ms": 284.41}, "triton_wait": {"avg_ms": 67.927, "count": 117, "max_ms": 96.417, "p95_ms": 89.548}, "visualize": {"avg_ms": 11.672, "count": 117, "max_ms": 14.821, "p95_ms": 13.352}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 311}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3974.723, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:12:24+09:00", "last_error_summary": "[h264 @ 0x55d37dc15280] error while decoding MB 67 11, bytestream -5", "last_exit_at": "2026-07-01T17:12:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.347, "last_started_at": "2026-07-01T17:10:33+09:00", "pid": 11300, "process": {"exists": false, "pid": 11300}, "restart_count": 71, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.358, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:12:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:12:24+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T17:12:24+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T17:12:24.193001+09:00", "mtime_epoch": 1782893544.193001, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170138}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:12:24+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 874.234, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.901, "pushed": 3.901, "read": 3.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.004, "count": 117, "max_ms": 18.012, "p95_ms": 11.531}, "gstreamer_push": {"avg_ms": 0.886, "count": 117, "max_ms": 1.828, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.637, "count": 117, "max_ms": 69.432, "p95_ms": 59.831}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.348, "count": 117, "max_ms": 89.881, "p95_ms": 88.564}, "yolo11_640_fire_11class": {"avg_ms": 53.595, "count": 117, "max_ms": 76.238, "p95_ms": 71.337}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.534, "count": 117, "max_ms": 96.404, "p95_ms": 88.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.25, "count": 117, "max_ms": 307.938, "p95_ms": 271.206}, "read_interval": {"avg_ms": 256.541, "count": 117, "max_ms": 470.911, "p95_ms": 284.41}, "triton_wait": {"avg_ms": 67.927, "count": 117, "max_ms": 96.417, "p95_ms": 89.548}, "visualize": {"avg_ms": 11.672, "count": 117, "max_ms": 14.821, "p95_ms": 13.352}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 311}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4079.992, "limit_mb": 8796093022207.996}, "detail": {"pid": 11407, "restart_count": 72, "start_count": 73}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:12:24+09:00", "last_error_summary": "[h264 @ 0x55d37dc15280] error while decoding MB 67 11, bytestream -5", "last_exit_at": "2026-07-01T17:12:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:12:27+09:00", "pid": 11407, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.648, "name": "ffmpeg", "pid": 11407, "rss_mb": 138.648, "state": "S (sleeping)", "threads": 106}, "restart_count": 72, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.358, "sample_count": 1121, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:12:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.938, "last_frame_pushed_at": "2026-07-01T17:12:24+09:00", "last_frame_read_age_sec": 5.164, "last_frame_read_at": "2026-07-01T17:12:24+09:00", "mjpeg": {"age_sec": 4.919, "exists": true, "mtime": "2026-07-01T17:12:24.429000+09:00", "mtime_epoch": 1782893544.4290001, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170638}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:12:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 874.234, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.283, "pushed": 3.283, "read": 3.25}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.073, "count": 97, "max_ms": 18.012, "p95_ms": 11.577}, "gstreamer_push": {"avg_ms": 0.883, "count": 98, "max_ms": 1.828, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.055, "count": 97, "max_ms": 68.644, "p95_ms": 59.831}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.931, "count": 97, "max_ms": 89.881, "p95_ms": 88.564}, "yolo11_640_fire_11class": {"avg_ms": 54.354, "count": 97, "max_ms": 76.238, "p95_ms": 72.077}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.294, "count": 97, "max_ms": 96.404, "p95_ms": 88.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.56, "count": 98, "max_ms": 307.938, "p95_ms": 271.206}, "read_interval": {"avg_ms": 257.205, "count": 97, "max_ms": 470.911, "p95_ms": 285.01}, "triton_wait": {"avg_ms": 68.706, "count": 97, "max_ms": 96.417, "p95_ms": 89.548}, "visualize": {"avg_ms": 11.669, "count": 97, "max_ms": 14.821, "p95_ms": 13.934}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.849, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3984.508, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56137d252600] error while decoding MB 22 8, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:14:20+09:00", "last_error_summary": "[h264 @ 0x56137d252600] error while decoding MB 22 8, bytestream -7", "last_exit_at": "2026-07-01T17:12:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.863, "last_started_at": "2026-07-01T17:12:27+09:00", "pid": 11407, "process": {"exists": false, "pid": 11407}, "restart_count": 72, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.877, "sample_count": 1127, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37764, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:14:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T17:14:20+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T17:14:20+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T17:14:20.164844+09:00", "mtime_epoch": 1782893660.1648436, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181835}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:14:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 879.516, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.706, "pushed": 3.706, "read": 3.706}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.066, "count": 110, "max_ms": 11.496, "p95_ms": 10.792}, "gstreamer_push": {"avg_ms": 0.945, "count": 111, "max_ms": 2.823, "p95_ms": 1.8}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.808, "count": 110, "max_ms": 72.242, "p95_ms": 69.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.41, "count": 110, "max_ms": 97.847, "p95_ms": 89.012}, "yolo11_640_fire_11class": {"avg_ms": 59.715, "count": 110, "max_ms": 78.684, "p95_ms": 73.262}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.983, "count": 110, "max_ms": 96.222, "p95_ms": 91.618}}, "truncated_count": 0}, "processing_total": {"avg_ms": 261.617, "count": 111, "max_ms": 352.52, "p95_ms": 334.576}, "read_interval": {"avg_ms": 271.388, "count": 111, "max_ms": 362.998, "p95_ms": 343.599}, "triton_wait": {"avg_ms": 75.4, "count": 110, "max_ms": 97.859, "p95_ms": 91.632}, "visualize": {"avg_ms": 11.77, "count": 110, "max_ms": 14.736, "p95_ms": 12.734}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3982.992, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:14:20+09:00", "last_error_summary": "[h264 @ 0x56137d252600] error while decoding MB 22 8, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:14:20+09:00", "last_error_summary": "[h264 @ 0x56137d252600] error while decoding MB 22 8, bytestream -7", "last_exit_at": "2026-07-01T17:14:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.083, "last_started_at": "2026-07-01T17:12:27+09:00", "pid": 11407, "process": {"exists": false, "pid": 11407}, "restart_count": 72, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.877, "sample_count": 1127, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37765, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:14:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:14:20+09:00", "last_frame_read_age_sec": 0.273, "last_frame_read_at": "2026-07-01T17:14:20+09:00", "mjpeg": {"age_sec": 0.265, "exists": true, "mtime": "2026-07-01T17:14:20.164844+09:00", "mtime_epoch": 1782893660.1648436, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181835}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:14:20+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 879.516, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.709, "pushed": 3.709, "read": 3.676}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.073, "count": 110, "max_ms": 11.496, "p95_ms": 10.792}, "gstreamer_push": {"avg_ms": 0.944, "count": 111, "max_ms": 2.823, "p95_ms": 1.8}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.834, "count": 110, "max_ms": 72.242, "p95_ms": 69.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.578, "count": 110, "max_ms": 97.847, "p95_ms": 89.012}, "yolo11_640_fire_11class": {"avg_ms": 59.885, "count": 110, "max_ms": 78.684, "p95_ms": 73.262}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.183, "count": 110, "max_ms": 96.222, "p95_ms": 91.618}}, "truncated_count": 0}, "processing_total": {"avg_ms": 261.993, "count": 111, "max_ms": 352.52, "p95_ms": 334.576}, "read_interval": {"avg_ms": 271.706, "count": 110, "max_ms": 362.998, "p95_ms": 343.599}, "triton_wait": {"avg_ms": 75.6, "count": 110, "max_ms": 97.859, "p95_ms": 91.632}, "visualize": {"avg_ms": 11.778, "count": 110, "max_ms": 14.736, "p95_ms": 12.734}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3983.207, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:14:20+09:00", "last_error_summary": "[h264 @ 0x56137d252600] error while decoding MB 22 8, bytestream -7", "last_exit_at": "2026-07-01T17:14:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.086, "last_started_at": "2026-07-01T17:12:27+09:00", "pid": 11407, "process": {"exists": false, "pid": 11407}, "restart_count": 72, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.877, "sample_count": 1127, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37765, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:14:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:14:20+09:00", "last_frame_read_age_sec": 0.276, "last_frame_read_at": "2026-07-01T17:14:20+09:00", "mjpeg": {"age_sec": 0.268, "exists": true, "mtime": "2026-07-01T17:14:20.164844+09:00", "mtime_epoch": 1782893660.1648436, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181835}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:14:20+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 879.516, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.709, "pushed": 3.709, "read": 3.675}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.073, "count": 110, "max_ms": 11.496, "p95_ms": 10.792}, "gstreamer_push": {"avg_ms": 0.944, "count": 111, "max_ms": 2.823, "p95_ms": 1.8}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.834, "count": 110, "max_ms": 72.242, "p95_ms": 69.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.578, "count": 110, "max_ms": 97.847, "p95_ms": 89.012}, "yolo11_640_fire_11class": {"avg_ms": 59.885, "count": 110, "max_ms": 78.684, "p95_ms": 73.262}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.183, "count": 110, "max_ms": 96.222, "p95_ms": 91.618}}, "truncated_count": 0}, "processing_total": {"avg_ms": 261.993, "count": 111, "max_ms": 352.52, "p95_ms": 334.576}, "read_interval": {"avg_ms": 271.706, "count": 110, "max_ms": 362.998, "p95_ms": 343.599}, "triton_wait": {"avg_ms": 75.6, "count": 110, "max_ms": 97.859, "p95_ms": 91.632}, "visualize": {"avg_ms": 11.778, "count": 110, "max_ms": 14.736, "p95_ms": 12.734}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 314}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4080.734, "limit_mb": 8796093022207.996}, "detail": {"pid": 11514, "restart_count": 73, "start_count": 74}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:14:20+09:00", "last_error_summary": "[h264 @ 0x56137d252600] error while decoding MB 22 8, bytestream -7", "last_exit_at": "2026-07-01T17:14:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:14:22+09:00", "pid": 11514, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.754, "name": "ffmpeg", "pid": 11514, "rss_mb": 133.754, "state": "S (sleeping)", "threads": 106}, "restart_count": 73, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.047, "sample_count": 1109, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 37765, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:14:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.422, "last_frame_pushed_at": "2026-07-01T17:14:20+09:00", "last_frame_read_age_sec": 4.692, "last_frame_read_at": "2026-07-01T17:14:20+09:00", "mjpeg": {"age_sec": 4.409, "exists": true, "mtime": "2026-07-01T17:14:20.440844+09:00", "mtime_epoch": 1782893660.4408436, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181981}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:14:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 879.516, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.157, "pushed": 3.157, "read": 3.123}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.018, "count": 94, "max_ms": 11.496, "p95_ms": 10.792}, "gstreamer_push": {"avg_ms": 0.944, "count": 94, "max_ms": 2.823, "p95_ms": 1.806}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.898, "count": 94, "max_ms": 72.242, "p95_ms": 69.911}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.46, "count": 94, "max_ms": 97.847, "p95_ms": 89.258}, "yolo11_640_fire_11class": {"avg_ms": 60.414, "count": 94, "max_ms": 78.684, "p95_ms": 73.72}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.055, "count": 94, "max_ms": 96.222, "p95_ms": 91.971}}, "truncated_count": 0}, "processing_total": {"avg_ms": 262.659, "count": 94, "max_ms": 349.66, "p95_ms": 334.576}, "read_interval": {"avg_ms": 272.376, "count": 93, "max_ms": 359.973, "p95_ms": 343.599}, "triton_wait": {"avg_ms": 76.524, "count": 94, "max_ms": 97.859, "p95_ms": 91.983}, "visualize": {"avg_ms": 11.791, "count": 94, "max_ms": 14.736, "p95_ms": 12.926}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.778, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 318}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3976.641, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56363de37c00] error while decoding MB 62 29, bytestream -31 [h264 @ 0x56363dd2da00] error while decoding MB 61 23, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:16:13+09:00", "last_error_summary": "[h264 @ 0x56363de37c00] error while decoding MB 62 29, bytestream -31 [h264 @ 0x56363dd2da00] error while decoding MB 61 23, bytestream -5", "last_exit_at": "2026-07-01T17:14:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.381, "last_started_at": "2026-07-01T17:14:22+09:00", "pid": 11514, "process": {"exists": false, "pid": 11514}, "restart_count": 73, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.275, "sample_count": 1130, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38185, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:16:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-01T17:16:13+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-01T17:16:13+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T17:16:13.185098+09:00", "mtime_epoch": 1782893773.1850977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180738}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:16:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 876.75, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.992, "pushed": 3.992, "read": 3.992}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.835, "count": 119, "max_ms": 12.546, "p95_ms": 10.952}, "gstreamer_push": {"avg_ms": 0.904, "count": 119, "max_ms": 3.053, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.022, "count": 119, "max_ms": 66.942, "p95_ms": 61.977}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.061, "count": 119, "max_ms": 92.976, "p95_ms": 87.945}, "yolo11_640_fire_11class": {"avg_ms": 52.042, "count": 119, "max_ms": 74.862, "p95_ms": 72.45}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.264, "count": 119, "max_ms": 91.735, "p95_ms": 88.475}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.643, "count": 119, "max_ms": 342.427, "p95_ms": 273.542}, "read_interval": {"avg_ms": 252.024, "count": 119, "max_ms": 353.514, "p95_ms": 282.669}, "triton_wait": {"avg_ms": 67.037, "count": 119, "max_ms": 92.988, "p95_ms": 88.57}, "visualize": {"avg_ms": 11.432, "count": 119, "max_ms": 13.656, "p95_ms": 12.565}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.808, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 318}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3975.691, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:16:13+09:00", "last_error_summary": "[h264 @ 0x56363de37c00] error while decoding MB 62 29, bytestream -31 [h264 @ 0x56363dd2da00] error while decoding MB 61 23, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:16:13+09:00", "last_error_summary": "[h264 @ 0x56363de37c00] error while decoding MB 62 29, bytestream -31 [h264 @ 0x56363dd2da00] error while decoding MB 61 23, bytestream -5", "last_exit_at": "2026-07-01T17:16:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.581, "last_started_at": "2026-07-01T17:14:22+09:00", "pid": 11514, "process": {"exists": false, "pid": 11514}, "restart_count": 73, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.275, "sample_count": 1130, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38186, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:16:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:16:13+09:00", "last_frame_read_age_sec": 0.257, "last_frame_read_at": "2026-07-01T17:16:13+09:00", "mjpeg": {"age_sec": 0.245, "exists": true, "mtime": "2026-07-01T17:16:13.185098+09:00", "mtime_epoch": 1782893773.1850977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180738}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:16:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 876.75, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.967, "pushed": 3.967, "read": 3.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.833, "count": 119, "max_ms": 12.546, "p95_ms": 10.952}, "gstreamer_push": {"avg_ms": 0.903, "count": 119, "max_ms": 3.053, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.15, "count": 119, "max_ms": 66.942, "p95_ms": 61.977}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.159, "count": 119, "max_ms": 92.976, "p95_ms": 87.945}, "yolo11_640_fire_11class": {"avg_ms": 52.169, "count": 119, "max_ms": 74.862, "p95_ms": 72.45}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.363, "count": 119, "max_ms": 91.735, "p95_ms": 88.475}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.755, "count": 119, "max_ms": 342.427, "p95_ms": 273.542}, "read_interval": {"avg_ms": 252.024, "count": 119, "max_ms": 353.514, "p95_ms": 282.669}, "triton_wait": {"avg_ms": 67.136, "count": 119, "max_ms": 92.988, "p95_ms": 88.57}, "visualize": {"avg_ms": 11.43, "count": 119, "max_ms": 13.656, "p95_ms": 12.565}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 318}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3975.684, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:16:13+09:00", "last_error_summary": "[h264 @ 0x56363de37c00] error while decoding MB 62 29, bytestream -31 [h264 @ 0x56363dd2da00] error while decoding MB 61 23, bytestream -5", "last_exit_at": "2026-07-01T17:16:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.585, "last_started_at": "2026-07-01T17:14:22+09:00", "pid": 11514, "process": {"exists": false, "pid": 11514}, "restart_count": 73, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.275, "sample_count": 1130, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38186, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:16:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:16:13+09:00", "last_frame_read_age_sec": 0.261, "last_frame_read_at": "2026-07-01T17:16:13+09:00", "mjpeg": {"age_sec": 0.248, "exists": true, "mtime": "2026-07-01T17:16:13.185098+09:00", "mtime_epoch": 1782893773.1850977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180738}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:16:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 876.75, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.998, "pushed": 3.998, "read": 3.965}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.833, "count": 119, "max_ms": 12.546, "p95_ms": 10.952}, "gstreamer_push": {"avg_ms": 0.903, "count": 119, "max_ms": 3.053, "p95_ms": 0.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.15, "count": 119, "max_ms": 66.942, "p95_ms": 61.977}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.159, "count": 119, "max_ms": 92.976, "p95_ms": 87.945}, "yolo11_640_fire_11class": {"avg_ms": 52.169, "count": 119, "max_ms": 74.862, "p95_ms": 72.45}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.363, "count": 119, "max_ms": 91.735, "p95_ms": 88.475}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.755, "count": 119, "max_ms": 342.427, "p95_ms": 273.542}, "read_interval": {"avg_ms": 252.048, "count": 118, "max_ms": 353.514, "p95_ms": 282.669}, "triton_wait": {"avg_ms": 67.136, "count": 119, "max_ms": 92.988, "p95_ms": 88.57}, "visualize": {"avg_ms": 11.43, "count": 119, "max_ms": 13.656, "p95_ms": 12.565}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.761, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 318}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4086.777, "limit_mb": 8796093022207.996}, "detail": {"pid": 11621, "restart_count": 74, "start_count": 75}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:16:13+09:00", "last_error_summary": "[h264 @ 0x56363de37c00] error while decoding MB 62 29, bytestream -31 [h264 @ 0x56363dd2da00] error while decoding MB 61 23, bytestream -5", "last_exit_at": "2026-07-01T17:16:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:16:15+09:00", "pid": 11621, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.523, "name": "ffmpeg", "pid": 11621, "rss_mb": 138.523, "state": "S (sleeping)", "threads": 106}, "restart_count": 74, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.37, "sample_count": 1113, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38186, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:16:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.383, "last_frame_pushed_at": "2026-07-01T17:16:13+09:00", "last_frame_read_age_sec": 4.637, "last_frame_read_at": "2026-07-01T17:16:13+09:00", "mjpeg": {"age_sec": 4.365, "exists": true, "mtime": "2026-07-01T17:16:13.445099+09:00", "mtime_epoch": 1782893773.4450986, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181333}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:16:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 876.75, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.444, "pushed": 3.444, "read": 3.411}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.827, "count": 102, "max_ms": 12.546, "p95_ms": 10.952}, "gstreamer_push": {"avg_ms": 0.898, "count": 103, "max_ms": 3.053, "p95_ms": 0.89}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.652, "count": 102, "max_ms": 66.942, "p95_ms": 61.258}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.215, "count": 102, "max_ms": 92.976, "p95_ms": 87.072}, "yolo11_640_fire_11class": {"avg_ms": 52.427, "count": 102, "max_ms": 74.862, "p95_ms": 70.103}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.342, "count": 102, "max_ms": 91.735, "p95_ms": 86.84}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.826, "count": 103, "max_ms": 342.427, "p95_ms": 269.567}, "read_interval": {"avg_ms": 250.045, "count": 102, "max_ms": 353.514, "p95_ms": 279.122}, "triton_wait": {"avg_ms": 67.22, "count": 102, "max_ms": 92.988, "p95_ms": 88.353}, "visualize": {"avg_ms": 11.374, "count": 102, "max_ms": 13.656, "p95_ms": 12.453}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 321}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3977.766, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55aa10da3440] error while decoding MB 29 39, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:18:08+09:00", "last_error_summary": "[h264 @ 0x55aa10da3440] error while decoding MB 29 39, bytestream -5", "last_exit_at": "2026-07-01T17:16:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.142, "last_started_at": "2026-07-01T17:16:15+09:00", "pid": 11621, "process": {"exists": false, "pid": 11621}, "restart_count": 74, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 279.918, "sample_count": 1115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38607, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:18:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-01T17:18:08+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-01T17:18:08+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-01T17:18:08.901691+09:00", "mtime_epoch": 1782893888.9016907, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186582}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:18:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 874.242, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.79, "pushed": 3.79, "read": 3.79}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.218, "count": 113, "max_ms": 16.505, "p95_ms": 11.59}, "gstreamer_push": {"avg_ms": 0.902, "count": 113, "max_ms": 1.872, "p95_ms": 1.09}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.437, "count": 113, "max_ms": 71.189, "p95_ms": 67.715}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.86, "count": 113, "max_ms": 93.291, "p95_ms": 88.134}, "yolo11_640_fire_11class": {"avg_ms": 57.557, "count": 113, "max_ms": 77.615, "p95_ms": 74.38}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.702, "count": 113, "max_ms": 93.287, "p95_ms": 90.877}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.901, "count": 113, "max_ms": 335.107, "p95_ms": 273.934}, "read_interval": {"avg_ms": 265.492, "count": 113, "max_ms": 345.251, "p95_ms": 284.655}, "triton_wait": {"avg_ms": 73.351, "count": 113, "max_ms": 93.305, "p95_ms": 90.923}, "visualize": {"avg_ms": 11.791, "count": 113, "max_ms": 13.181, "p95_ms": 12.469}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.813, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 321}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3977.016, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:18:08+09:00", "last_error_summary": "[h264 @ 0x55aa10da3440] error while decoding MB 29 39, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:18:08+09:00", "last_error_summary": "[h264 @ 0x55aa10da3440] error while decoding MB 29 39, bytestream -5", "last_exit_at": "2026-07-01T17:18:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.341, "last_started_at": "2026-07-01T17:16:15+09:00", "pid": 11621, "process": {"exists": false, "pid": 11621}, "restart_count": 74, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 279.918, "sample_count": 1115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38608, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:18:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:18:09+09:00", "last_frame_read_age_sec": 0.258, "last_frame_read_at": "2026-07-01T17:18:08+09:00", "mjpeg": {"age_sec": 0.249, "exists": true, "mtime": "2026-07-01T17:18:08.901691+09:00", "mtime_epoch": 1782893888.9016907, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186582}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:18:09+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 874.242, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.798, "pushed": 3.798, "read": 3.765}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.217, "count": 113, "max_ms": 16.505, "p95_ms": 11.59}, "gstreamer_push": {"avg_ms": 0.902, "count": 113, "max_ms": 1.872, "p95_ms": 1.09}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.404, "count": 113, "max_ms": 71.189, "p95_ms": 67.715}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.824, "count": 113, "max_ms": 93.291, "p95_ms": 88.134}, "yolo11_640_fire_11class": {"avg_ms": 57.534, "count": 113, "max_ms": 77.615, "p95_ms": 74.38}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.666, "count": 113, "max_ms": 93.287, "p95_ms": 90.877}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.95, "count": 113, "max_ms": 335.107, "p95_ms": 273.934}, "read_interval": {"avg_ms": 265.566, "count": 112, "max_ms": 345.251, "p95_ms": 284.655}, "triton_wait": {"avg_ms": 73.315, "count": 113, "max_ms": 93.305, "p95_ms": 90.923}, "visualize": {"avg_ms": 11.802, "count": 113, "max_ms": 13.74, "p95_ms": 12.614}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.751, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 321}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3977.008, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:18:08+09:00", "last_error_summary": "[h264 @ 0x55aa10da3440] error while decoding MB 29 39, bytestream -5", "last_exit_at": "2026-07-01T17:18:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.345, "last_started_at": "2026-07-01T17:16:15+09:00", "pid": 11621, "process": {"exists": false, "pid": 11621}, "restart_count": 74, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 279.918, "sample_count": 1115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38608, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:18:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:18:09+09:00", "last_frame_read_age_sec": 0.261, "last_frame_read_at": "2026-07-01T17:18:08+09:00", "mjpeg": {"age_sec": 0.252, "exists": true, "mtime": "2026-07-01T17:18:08.901691+09:00", "mtime_epoch": 1782893888.9016907, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186582}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:18:09+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 874.242, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.798, "pushed": 3.798, "read": 3.764}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.217, "count": 113, "max_ms": 16.505, "p95_ms": 11.59}, "gstreamer_push": {"avg_ms": 0.902, "count": 113, "max_ms": 1.872, "p95_ms": 1.09}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.404, "count": 113, "max_ms": 71.189, "p95_ms": 67.715}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.824, "count": 113, "max_ms": 93.291, "p95_ms": 88.134}, "yolo11_640_fire_11class": {"avg_ms": 57.534, "count": 113, "max_ms": 77.615, "p95_ms": 74.38}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.666, "count": 113, "max_ms": 93.287, "p95_ms": 90.877}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.95, "count": 113, "max_ms": 335.107, "p95_ms": 273.934}, "read_interval": {"avg_ms": 265.566, "count": 112, "max_ms": 345.251, "p95_ms": 284.655}, "triton_wait": {"avg_ms": 73.315, "count": 113, "max_ms": 93.305, "p95_ms": 90.923}, "visualize": {"avg_ms": 11.802, "count": 113, "max_ms": 13.74, "p95_ms": 12.614}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.754, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 321}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4081.855, "limit_mb": 8796093022207.996}, "detail": {"pid": 11728, "restart_count": 75, "start_count": 76}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:18:08+09:00", "last_error_summary": "[h264 @ 0x55aa10da3440] error while decoding MB 29 39, bytestream -5", "last_exit_at": "2026-07-01T17:18:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:18:11+09:00", "pid": 11728, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.57, "name": "ffmpeg", "pid": 11728, "rss_mb": 136.57, "state": "S (sleeping)", "threads": 106}, "restart_count": 75, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 279.918, "sample_count": 1096, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 38608, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:18:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.796, "last_frame_pushed_at": "2026-07-01T17:18:09+09:00", "last_frame_read_age_sec": 5.051, "last_frame_read_at": "2026-07-01T17:18:08+09:00", "mjpeg": {"age_sec": 4.778, "exists": true, "mtime": "2026-07-01T17:18:09.165693+09:00", "mtime_epoch": 1782893889.1656926, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186438}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:18:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 874.242, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.172, "pushed": 3.172, "read": 3.139}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.992, "count": 94, "max_ms": 11.611, "p95_ms": 11.035}, "gstreamer_push": {"avg_ms": 0.91, "count": 95, "max_ms": 1.872, "p95_ms": 1.769}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.515, "count": 94, "max_ms": 71.189, "p95_ms": 68.6}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.177, "count": 94, "max_ms": 93.291, "p95_ms": 88.187}, "yolo11_640_fire_11class": {"avg_ms": 58.833, "count": 94, "max_ms": 77.615, "p95_ms": 74.746}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.1, "count": 94, "max_ms": 93.287, "p95_ms": 90.911}}, "truncated_count": 0}, "processing_total": {"avg_ms": 257.76, "count": 95, "max_ms": 335.107, "p95_ms": 275.156}, "read_interval": {"avg_ms": 267.397, "count": 94, "max_ms": 345.251, "p95_ms": 285.537}, "triton_wait": {"avg_ms": 74.815, "count": 94, "max_ms": 93.305, "p95_ms": 92.522}, "visualize": {"avg_ms": 11.809, "count": 94, "max_ms": 13.74, "p95_ms": 12.371}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 325}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3981.348, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ebbda0d2c0] error while decoding MB 71 9, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:20:01+09:00", "last_error_summary": "[h264 @ 0x55ebbda0d2c0] error while decoding MB 71 9, bytestream -7", "last_exit_at": "2026-07-01T17:18:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.913, "last_started_at": "2026-07-01T17:18:11+09:00", "pid": 11728, "process": {"exists": false, "pid": 11728}, "restart_count": 75, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.453, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39026, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:20:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T17:20:01+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T17:20:01+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T17:20:01.814530+09:00", "mtime_epoch": 1782894001.8145297, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187647}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:20:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 877.785, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.956, "pushed": 3.956, "read": 3.956}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.975, "count": 118, "max_ms": 15.366, "p95_ms": 10.936}, "gstreamer_push": {"avg_ms": 0.927, "count": 118, "max_ms": 2.897, "p95_ms": 1.784}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.151, "count": 118, "max_ms": 72.252, "p95_ms": 67.841}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.233, "count": 118, "max_ms": 92.104, "p95_ms": 87.721}, "yolo11_640_fire_11class": {"avg_ms": 52.04, "count": 118, "max_ms": 75.546, "p95_ms": 71.553}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.211, "count": 118, "max_ms": 91.927, "p95_ms": 89.629}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.608, "count": 118, "max_ms": 302.702, "p95_ms": 274.462}, "read_interval": {"avg_ms": 254.309, "count": 118, "max_ms": 314.441, "p95_ms": 283.726}, "triton_wait": {"avg_ms": 66.454, "count": 118, "max_ms": 92.117, "p95_ms": 90.377}, "visualize": {"avg_ms": 11.516, "count": 118, "max_ms": 13.215, "p95_ms": 12.622}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.825, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 325}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3980.238, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:20:01+09:00", "last_error_summary": "[h264 @ 0x55ebbda0d2c0] error while decoding MB 71 9, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:20:01+09:00", "last_error_summary": "[h264 @ 0x55ebbda0d2c0] error while decoding MB 71 9, bytestream -7", "last_exit_at": "2026-07-01T17:20:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.098, "last_started_at": "2026-07-01T17:18:11+09:00", "pid": 11728, "process": {"exists": false, "pid": 11728}, "restart_count": 75, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.453, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39027, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:20:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:20:02+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-01T17:20:01+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-01T17:20:01.814530+09:00", "mtime_epoch": 1782894001.8145297, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187647}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:20:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 877.785, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.962, "pushed": 3.962, "read": 3.928}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.975, "count": 118, "max_ms": 15.366, "p95_ms": 10.936}, "gstreamer_push": {"avg_ms": 0.927, "count": 118, "max_ms": 2.897, "p95_ms": 1.784}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.16, "count": 118, "max_ms": 72.252, "p95_ms": 67.841}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.181, "count": 118, "max_ms": 92.104, "p95_ms": 87.721}, "yolo11_640_fire_11class": {"avg_ms": 51.996, "count": 118, "max_ms": 75.546, "p95_ms": 71.553}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.155, "count": 118, "max_ms": 91.927, "p95_ms": 89.629}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.584, "count": 118, "max_ms": 302.702, "p95_ms": 274.462}, "read_interval": {"avg_ms": 254.401, "count": 117, "max_ms": 314.441, "p95_ms": 283.726}, "triton_wait": {"avg_ms": 66.397, "count": 118, "max_ms": 92.117, "p95_ms": 90.377}, "visualize": {"avg_ms": 11.517, "count": 118, "max_ms": 13.215, "p95_ms": 12.622}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.782, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 325}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3980.23, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:20:01+09:00", "last_error_summary": "[h264 @ 0x55ebbda0d2c0] error while decoding MB 71 9, bytestream -7", "last_exit_at": "2026-07-01T17:20:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.101, "last_started_at": "2026-07-01T17:18:11+09:00", "pid": 11728, "process": {"exists": false, "pid": 11728}, "restart_count": 75, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.453, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39027, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:20:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:20:02+09:00", "last_frame_read_age_sec": 0.239, "last_frame_read_at": "2026-07-01T17:20:01+09:00", "mjpeg": {"age_sec": 0.229, "exists": true, "mtime": "2026-07-01T17:20:01.814530+09:00", "mtime_epoch": 1782894001.8145297, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187647}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:20:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 877.785, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.962, "pushed": 3.962, "read": 3.928}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.975, "count": 118, "max_ms": 15.366, "p95_ms": 10.936}, "gstreamer_push": {"avg_ms": 0.927, "count": 118, "max_ms": 2.897, "p95_ms": 1.784}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.16, "count": 118, "max_ms": 72.252, "p95_ms": 67.841}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.181, "count": 118, "max_ms": 92.104, "p95_ms": 87.721}, "yolo11_640_fire_11class": {"avg_ms": 51.996, "count": 118, "max_ms": 75.546, "p95_ms": 71.553}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.155, "count": 118, "max_ms": 91.927, "p95_ms": 89.629}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.584, "count": 118, "max_ms": 302.702, "p95_ms": 274.462}, "read_interval": {"avg_ms": 254.401, "count": 117, "max_ms": 314.441, "p95_ms": 283.726}, "triton_wait": {"avg_ms": 66.397, "count": 118, "max_ms": 92.117, "p95_ms": 90.377}, "visualize": {"avg_ms": 11.517, "count": 118, "max_ms": 13.215, "p95_ms": 12.622}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.786, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 325}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4088.953, "limit_mb": 8796093022207.996}, "detail": {"pid": 11835, "restart_count": 76, "start_count": 77}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:20:01+09:00", "last_error_summary": "[h264 @ 0x55ebbda0d2c0] error while decoding MB 71 9, bytestream -7", "last_exit_at": "2026-07-01T17:20:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:20:04+09:00", "pid": 11835, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.586, "name": "ffmpeg", "pid": 11835, "rss_mb": 136.586, "state": "S (sleeping)", "threads": 106}, "restart_count": 76, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.511, "sample_count": 1106, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39027, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:20:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.164, "last_frame_pushed_at": "2026-07-01T17:20:02+09:00", "last_frame_read_age_sec": 4.397, "last_frame_read_at": "2026-07-01T17:20:01+09:00", "mjpeg": {"age_sec": 4.147, "exists": true, "mtime": "2026-07-01T17:20:02.054532+09:00", "mtime_epoch": 1782894002.0545316, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187698}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:20:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 877.785, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.414, "pushed": 3.414, "read": 3.38}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.98, "count": 101, "max_ms": 15.366, "p95_ms": 10.936}, "gstreamer_push": {"avg_ms": 0.929, "count": 102, "max_ms": 2.897, "p95_ms": 1.058}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.673, "count": 101, "max_ms": 72.252, "p95_ms": 67.841}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.465, "count": 101, "max_ms": 92.104, "p95_ms": 87.721}, "yolo11_640_fire_11class": {"avg_ms": 52.394, "count": 101, "max_ms": 75.546, "p95_ms": 71.553}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.535, "count": 101, "max_ms": 91.927, "p95_ms": 89.629}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.804, "count": 102, "max_ms": 302.702, "p95_ms": 274.462}, "read_interval": {"avg_ms": 254.657, "count": 101, "max_ms": 314.441, "p95_ms": 283.726}, "triton_wait": {"avg_ms": 66.784, "count": 101, "max_ms": 92.117, "p95_ms": 90.377}, "visualize": {"avg_ms": 11.473, "count": 101, "max_ms": 13.215, "p95_ms": 12.372}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 328}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3978.586, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f27db24cc0] error while decoding MB 64 28, bytestream -18", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:21:55+09:00", "last_error_summary": "[h264 @ 0x55f27db24cc0] error while decoding MB 64 28, bytestream -18", "last_exit_at": "2026-07-01T17:20:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.32, "last_started_at": "2026-07-01T17:20:04+09:00", "pid": 11835, "process": {"exists": false, "pid": 11835}, "restart_count": 76, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.532, "sample_count": 1126, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39455, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:21:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T17:21:55+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T17:21:55+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T17:21:55.475581+09:00", "mtime_epoch": 1782894115.4755814, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:21:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 873.805, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.872, "pushed": 3.872, "read": 3.872}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.186, "count": 115, "max_ms": 15.616, "p95_ms": 11.87}, "gstreamer_push": {"avg_ms": 0.931, "count": 116, "max_ms": 1.953, "p95_ms": 1.797}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.093, "count": 115, "max_ms": 69.115, "p95_ms": 63.743}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.195, "count": 115, "max_ms": 91.434, "p95_ms": 89.279}, "yolo11_640_fire_11class": {"avg_ms": 53.64, "count": 115, "max_ms": 73.673, "p95_ms": 72.731}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.082, "count": 115, "max_ms": 92.657, "p95_ms": 89.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.569, "count": 116, "max_ms": 343.009, "p95_ms": 283.519}, "read_interval": {"avg_ms": 260.095, "count": 116, "max_ms": 354.134, "p95_ms": 293.159}, "triton_wait": {"avg_ms": 68.388, "count": 115, "max_ms": 92.669, "p95_ms": 90.855}, "visualize": {"avg_ms": 11.771, "count": 115, "max_ms": 17.889, "p95_ms": 12.698}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 328}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3980.996, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:21:55+09:00", "last_error_summary": "[h264 @ 0x55f27db24cc0] error while decoding MB 64 28, bytestream -18", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:21:55+09:00", "last_error_summary": "[h264 @ 0x55f27db24cc0] error while decoding MB 64 28, bytestream -18", "last_exit_at": "2026-07-01T17:21:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.521, "last_started_at": "2026-07-01T17:20:04+09:00", "pid": 11835, "process": {"exists": false, "pid": 11835}, "restart_count": 76, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.494, "sample_count": 1126, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:21:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:21:55+09:00", "last_frame_read_age_sec": 0.256, "last_frame_read_at": "2026-07-01T17:21:55+09:00", "mjpeg": {"age_sec": 0.246, "exists": true, "mtime": "2026-07-01T17:21:55.475581+09:00", "mtime_epoch": 1782894115.4755814, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:21:55+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 877.414, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.882, "pushed": 3.882, "read": 3.849}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.212, "count": 115, "max_ms": 15.616, "p95_ms": 12.419}, "gstreamer_push": {"avg_ms": 0.932, "count": 116, "max_ms": 1.953, "p95_ms": 1.797}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.047, "count": 115, "max_ms": 69.115, "p95_ms": 63.743}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.927, "count": 115, "max_ms": 91.434, "p95_ms": 88.931}, "yolo11_640_fire_11class": {"avg_ms": 53.409, "count": 115, "max_ms": 73.673, "p95_ms": 72.731}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.831, "count": 115, "max_ms": 92.657, "p95_ms": 89.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.42, "count": 116, "max_ms": 343.009, "p95_ms": 283.519}, "read_interval": {"avg_ms": 259.942, "count": 115, "max_ms": 354.134, "p95_ms": 293.159}, "triton_wait": {"avg_ms": 68.129, "count": 115, "max_ms": 92.669, "p95_ms": 90.855}, "visualize": {"avg_ms": 11.787, "count": 115, "max_ms": 17.889, "p95_ms": 13.201}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 328}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3980.988, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:21:55+09:00", "last_error_summary": "[h264 @ 0x55f27db24cc0] error while decoding MB 64 28, bytestream -18", "last_exit_at": "2026-07-01T17:21:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.524, "last_started_at": "2026-07-01T17:20:04+09:00", "pid": 11835, "process": {"exists": false, "pid": 11835}, "restart_count": 76, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.494, "sample_count": 1126, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:21:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:21:55+09:00", "last_frame_read_age_sec": 0.259, "last_frame_read_at": "2026-07-01T17:21:55+09:00", "mjpeg": {"age_sec": 0.249, "exists": true, "mtime": "2026-07-01T17:21:55.475581+09:00", "mtime_epoch": 1782894115.4755814, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:21:55+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 877.414, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.882, "pushed": 3.882, "read": 3.848}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.212, "count": 115, "max_ms": 15.616, "p95_ms": 12.419}, "gstreamer_push": {"avg_ms": 0.932, "count": 116, "max_ms": 1.953, "p95_ms": 1.797}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.047, "count": 115, "max_ms": 69.115, "p95_ms": 63.743}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.927, "count": 115, "max_ms": 91.434, "p95_ms": 88.931}, "yolo11_640_fire_11class": {"avg_ms": 53.409, "count": 115, "max_ms": 73.673, "p95_ms": 72.731}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.831, "count": 115, "max_ms": 92.657, "p95_ms": 89.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.42, "count": 116, "max_ms": 343.009, "p95_ms": 283.519}, "read_interval": {"avg_ms": 259.942, "count": 115, "max_ms": 354.134, "p95_ms": 293.159}, "triton_wait": {"avg_ms": 68.129, "count": 115, "max_ms": 92.669, "p95_ms": 90.855}, "visualize": {"avg_ms": 11.787, "count": 115, "max_ms": 17.889, "p95_ms": 13.201}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 328}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4092.328, "limit_mb": 8796093022207.996}, "detail": {"pid": 11942, "restart_count": 77, "start_count": 78}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:21:55+09:00", "last_error_summary": "[h264 @ 0x55f27db24cc0] error while decoding MB 64 28, bytestream -18", "last_exit_at": "2026-07-01T17:21:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:21:58+09:00", "pid": 11942, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.062, "name": "ffmpeg", "pid": 11942, "rss_mb": 142.062, "state": "S (sleeping)", "threads": 106}, "restart_count": 77, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.343, "sample_count": 1107, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:21:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.859, "last_frame_pushed_at": "2026-07-01T17:21:55+09:00", "last_frame_read_age_sec": 5.112, "last_frame_read_at": "2026-07-01T17:21:55+09:00", "mjpeg": {"age_sec": 4.838, "exists": true, "mtime": "2026-07-01T17:21:55.739584+09:00", "mtime_epoch": 1782894115.739584, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191706}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:22:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 877.414, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.277, "pushed": 3.277, "read": 3.243}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.011, "count": 97, "max_ms": 13.396, "p95_ms": 11.732}, "gstreamer_push": {"avg_ms": 0.922, "count": 98, "max_ms": 1.894, "p95_ms": 1.782}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.26, "count": 97, "max_ms": 69.115, "p95_ms": 63.743}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.54, "count": 97, "max_ms": 91.434, "p95_ms": 89.659}, "yolo11_640_fire_11class": {"avg_ms": 52.951, "count": 97, "max_ms": 73.673, "p95_ms": 72.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.327, "count": 97, "max_ms": 92.657, "p95_ms": 90.166}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.639, "count": 98, "max_ms": 343.009, "p95_ms": 283.519}, "read_interval": {"avg_ms": 258.112, "count": 97, "max_ms": 354.134, "p95_ms": 293.159}, "triton_wait": {"avg_ms": 67.666, "count": 97, "max_ms": 92.669, "p95_ms": 90.95}, "visualize": {"avg_ms": 11.729, "count": 97, "max_ms": 17.889, "p95_ms": 12.698}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 332}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3986.832, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562447b5ad40] error while decoding MB 3 39, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:23:54+09:00", "last_error_summary": "[h264 @ 0x562447b5ad40] error while decoding MB 3 39, bytestream -5", "last_exit_at": "2026-07-01T17:21:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.905, "last_started_at": "2026-07-01T17:21:58+09:00", "pid": 11942, "process": {"exists": false, "pid": 11942}, "restart_count": 77, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.235, "sample_count": 1131, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39891, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:23:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T17:23:54+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T17:23:54+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T17:23:54.436860+09:00", "mtime_epoch": 1782894234.4368603, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188879}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:23:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 880.375, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.802, "pushed": 3.802, "read": 3.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.034, "count": 113, "max_ms": 11.177, "p95_ms": 10.622}, "gstreamer_push": {"avg_ms": 0.948, "count": 114, "max_ms": 2.888, "p95_ms": 1.799}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.983, "count": 113, "max_ms": 69.659, "p95_ms": 61.75}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.841, "count": 113, "max_ms": 95.367, "p95_ms": 90.592}, "yolo11_640_fire_11class": {"avg_ms": 55.426, "count": 113, "max_ms": 77.845, "p95_ms": 72.16}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.035, "count": 113, "max_ms": 96.057, "p95_ms": 90.307}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.131, "count": 114, "max_ms": 347.67, "p95_ms": 276.536}, "read_interval": {"avg_ms": 264.751, "count": 114, "max_ms": 358.735, "p95_ms": 286.269}, "triton_wait": {"avg_ms": 71.489, "count": 113, "max_ms": 96.069, "p95_ms": 91.157}, "visualize": {"avg_ms": 11.622, "count": 113, "max_ms": 12.625, "p95_ms": 12.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 332}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3985.703, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:23:54+09:00", "last_error_summary": "[h264 @ 0x562447b5ad40] error while decoding MB 3 39, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:23:54+09:00", "last_error_summary": "[h264 @ 0x562447b5ad40] error while decoding MB 3 39, bytestream -5", "last_exit_at": "2026-07-01T17:23:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 116.127, "last_started_at": "2026-07-01T17:21:58+09:00", "pid": 11942, "process": {"exists": false, "pid": 11942}, "restart_count": 77, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.235, "sample_count": 1131, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39892, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:23:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:23:54+09:00", "last_frame_read_age_sec": 0.275, "last_frame_read_at": "2026-07-01T17:23:54+09:00", "mjpeg": {"age_sec": 0.267, "exists": true, "mtime": "2026-07-01T17:23:54.436860+09:00", "mtime_epoch": 1782894234.4368603, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188879}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:23:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 880.375, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.806, "pushed": 3.806, "read": 3.772}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.033, "count": 113, "max_ms": 11.177, "p95_ms": 10.622}, "gstreamer_push": {"avg_ms": 0.949, "count": 114, "max_ms": 2.888, "p95_ms": 1.799}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.236, "count": 113, "max_ms": 69.659, "p95_ms": 63.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.046, "count": 113, "max_ms": 95.367, "p95_ms": 90.592}, "yolo11_640_fire_11class": {"avg_ms": 55.687, "count": 113, "max_ms": 77.845, "p95_ms": 73.248}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.266, "count": 113, "max_ms": 96.057, "p95_ms": 90.307}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.334, "count": 114, "max_ms": 347.67, "p95_ms": 276.536}, "read_interval": {"avg_ms": 264.828, "count": 113, "max_ms": 358.735, "p95_ms": 286.269}, "triton_wait": {"avg_ms": 71.713, "count": 113, "max_ms": 96.069, "p95_ms": 91.157}, "visualize": {"avg_ms": 11.62, "count": 113, "max_ms": 12.625, "p95_ms": 12.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 332}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3985.816, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:23:54+09:00", "last_error_summary": "[h264 @ 0x562447b5ad40] error while decoding MB 3 39, bytestream -5", "last_exit_at": "2026-07-01T17:23:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 116.13, "last_started_at": "2026-07-01T17:21:58+09:00", "pid": 11942, "process": {"exists": false, "pid": 11942}, "restart_count": 77, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.235, "sample_count": 1131, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39892, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:23:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T17:23:54+09:00", "last_frame_read_age_sec": 0.278, "last_frame_read_at": "2026-07-01T17:23:54+09:00", "mjpeg": {"age_sec": 0.27, "exists": true, "mtime": "2026-07-01T17:23:54.436860+09:00", "mtime_epoch": 1782894234.4368603, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188879}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:23:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 880.375, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.805, "pushed": 3.805, "read": 3.772}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.033, "count": 113, "max_ms": 11.177, "p95_ms": 10.622}, "gstreamer_push": {"avg_ms": 0.949, "count": 114, "max_ms": 2.888, "p95_ms": 1.799}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.236, "count": 113, "max_ms": 69.659, "p95_ms": 63.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.046, "count": 113, "max_ms": 95.367, "p95_ms": 90.592}, "yolo11_640_fire_11class": {"avg_ms": 55.687, "count": 113, "max_ms": 77.845, "p95_ms": 73.248}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.266, "count": 113, "max_ms": 96.057, "p95_ms": 90.307}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.334, "count": 114, "max_ms": 347.67, "p95_ms": 276.536}, "read_interval": {"avg_ms": 264.828, "count": 113, "max_ms": 358.735, "p95_ms": 286.269}, "triton_wait": {"avg_ms": 71.713, "count": 113, "max_ms": 96.069, "p95_ms": 91.157}, "visualize": {"avg_ms": 11.62, "count": 113, "max_ms": 12.625, "p95_ms": 12.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 332}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4104.633, "limit_mb": 8796093022207.996}, "detail": {"pid": 12049, "restart_count": 78, "start_count": 79}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:23:54+09:00", "last_error_summary": "[h264 @ 0x562447b5ad40] error while decoding MB 3 39, bytestream -5", "last_exit_at": "2026-07-01T17:23:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:23:57+09:00", "pid": 12049, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.16, "name": "ffmpeg", "pid": 12049, "rss_mb": 147.16, "state": "S (sleeping)", "threads": 106}, "restart_count": 78, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.225, "sample_count": 1112, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 39892, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:23:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.886, "last_frame_pushed_at": "2026-07-01T17:23:54+09:00", "last_frame_read_age_sec": 5.157, "last_frame_read_at": "2026-07-01T17:23:54+09:00", "mjpeg": {"age_sec": 4.869, "exists": true, "mtime": "2026-07-01T17:23:54.716863+09:00", "mtime_epoch": 1782894234.7168634, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189039}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:23:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 884.465, "name": "celery", "pid": 237, "rss_mb": 880.375, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.191, "pushed": 3.191, "read": 3.157}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.019, "count": 95, "max_ms": 11.177, "p95_ms": 10.713}, "gstreamer_push": {"avg_ms": 0.954, "count": 95, "max_ms": 2.888, "p95_ms": 1.799}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.119, "count": 95, "max_ms": 69.659, "p95_ms": 63.923}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.179, "count": 95, "max_ms": 95.367, "p95_ms": 90.658}, "yolo11_640_fire_11class": {"avg_ms": 55.714, "count": 95, "max_ms": 77.845, "p95_ms": 73.248}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.363, "count": 95, "max_ms": 96.057, "p95_ms": 90.824}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.019, "count": 95, "max_ms": 347.67, "p95_ms": 275.535}, "read_interval": {"avg_ms": 264.491, "count": 94, "max_ms": 358.735, "p95_ms": 285.05}, "triton_wait": {"avg_ms": 71.867, "count": 95, "max_ms": 96.069, "p95_ms": 91.2}, "visualize": {"avg_ms": 11.583, "count": 95, "max_ms": 12.203, "p95_ms": 12.059}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.773, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3981.691, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56278497ec00] error while decoding MB 17 10, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:25:49+09:00", "last_error_summary": "[h264 @ 0x56278497ec00] error while decoding MB 17 10, bytestream -5", "last_exit_at": "2026-07-01T17:23:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.99, "last_started_at": "2026-07-01T17:23:57+09:00", "pid": 12049, "process": {"exists": false, "pid": 12049}, "restart_count": 78, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.375, "sample_count": 1115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40313, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:25:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T17:25:49+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T17:25:49+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T17:25:49.534236+09:00", "mtime_epoch": 1782894349.5342357, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:25:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 877.273, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.848, "pushed": 3.848, "read": 3.848}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.953, "count": 114, "max_ms": 11.197, "p95_ms": 10.866}, "gstreamer_push": {"avg_ms": 0.957, "count": 115, "max_ms": 2.887, "p95_ms": 1.853}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.142, "count": 114, "max_ms": 71.087, "p95_ms": 64.52}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.262, "count": 114, "max_ms": 91.509, "p95_ms": 88.401}, "yolo11_640_fire_11class": {"avg_ms": 53.845, "count": 114, "max_ms": 77.283, "p95_ms": 71.613}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.405, "count": 114, "max_ms": 90.774, "p95_ms": 89.806}}, "truncated_count": 0}, "processing_total": {"avg_ms": 251.802, "count": 115, "max_ms": 327.305, "p95_ms": 310.525}, "read_interval": {"avg_ms": 261.523, "count": 115, "max_ms": 338.56, "p95_ms": 322.441}, "triton_wait": {"avg_ms": 68.805, "count": 114, "max_ms": 91.52, "p95_ms": 89.856}, "visualize": {"avg_ms": 11.736, "count": 114, "max_ms": 15.071, "p95_ms": 12.636}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3980.66, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:25:49+09:00", "last_error_summary": "[h264 @ 0x56278497ec00] error while decoding MB 17 10, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:25:49+09:00", "last_error_summary": "[h264 @ 0x56278497ec00] error while decoding MB 17 10, bytestream -5", "last_exit_at": "2026-07-01T17:25:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.221, "last_started_at": "2026-07-01T17:23:57+09:00", "pid": 12049, "process": {"exists": false, "pid": 12049}, "restart_count": 78, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.419, "sample_count": 1115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40314, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:25:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:25:49+09:00", "last_frame_read_age_sec": 0.281, "last_frame_read_at": "2026-07-01T17:25:49+09:00", "mjpeg": {"age_sec": 0.272, "exists": true, "mtime": "2026-07-01T17:25:49.534236+09:00", "mtime_epoch": 1782894349.5342357, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:25:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 877.273, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.85, "pushed": 3.85, "read": 3.817}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.952, "count": 114, "max_ms": 11.197, "p95_ms": 10.866}, "gstreamer_push": {"avg_ms": 0.957, "count": 115, "max_ms": 2.887, "p95_ms": 1.853}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.17, "count": 114, "max_ms": 71.087, "p95_ms": 64.52}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.589, "count": 114, "max_ms": 95.08, "p95_ms": 88.534}, "yolo11_640_fire_11class": {"avg_ms": 54.035, "count": 114, "max_ms": 77.283, "p95_ms": 71.613}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.702, "count": 114, "max_ms": 92.551, "p95_ms": 89.842}}, "truncated_count": 0}, "processing_total": {"avg_ms": 252.143, "count": 115, "max_ms": 327.305, "p95_ms": 310.525}, "read_interval": {"avg_ms": 261.638, "count": 114, "max_ms": 338.56, "p95_ms": 322.441}, "triton_wait": {"avg_ms": 69.124, "count": 114, "max_ms": 95.093, "p95_ms": 90.105}, "visualize": {"avg_ms": 11.737, "count": 114, "max_ms": 15.071, "p95_ms": 12.636}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3980.652, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:25:49+09:00", "last_error_summary": "[h264 @ 0x56278497ec00] error while decoding MB 17 10, bytestream -5", "last_exit_at": "2026-07-01T17:25:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.224, "last_started_at": "2026-07-01T17:23:57+09:00", "pid": 12049, "process": {"exists": false, "pid": 12049}, "restart_count": 78, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.419, "sample_count": 1115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40314, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:25:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:25:49+09:00", "last_frame_read_age_sec": 0.284, "last_frame_read_at": "2026-07-01T17:25:49+09:00", "mjpeg": {"age_sec": 0.275, "exists": true, "mtime": "2026-07-01T17:25:49.534236+09:00", "mtime_epoch": 1782894349.5342357, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:25:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 877.273, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.85, "pushed": 3.85, "read": 3.817}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.952, "count": 114, "max_ms": 11.197, "p95_ms": 10.866}, "gstreamer_push": {"avg_ms": 0.957, "count": 115, "max_ms": 2.887, "p95_ms": 1.853}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.17, "count": 114, "max_ms": 71.087, "p95_ms": 64.52}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.589, "count": 114, "max_ms": 95.08, "p95_ms": 88.534}, "yolo11_640_fire_11class": {"avg_ms": 54.035, "count": 114, "max_ms": 77.283, "p95_ms": 71.613}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.702, "count": 114, "max_ms": 92.551, "p95_ms": 89.842}}, "truncated_count": 0}, "processing_total": {"avg_ms": 252.143, "count": 115, "max_ms": 327.305, "p95_ms": 310.525}, "read_interval": {"avg_ms": 261.638, "count": 114, "max_ms": 338.56, "p95_ms": 322.441}, "triton_wait": {"avg_ms": 69.124, "count": 114, "max_ms": 95.093, "p95_ms": 90.105}, "visualize": {"avg_ms": 11.737, "count": 114, "max_ms": 15.071, "p95_ms": 12.636}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4089.027, "limit_mb": 8796093022207.996}, "detail": {"pid": 12156, "restart_count": 79, "start_count": 80}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:25:49+09:00", "last_error_summary": "[h264 @ 0x56278497ec00] error while decoding MB 17 10, bytestream -5", "last_exit_at": "2026-07-01T17:25:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:25:52+09:00", "pid": 12156, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.695, "name": "ffmpeg", "pid": 12156, "rss_mb": 136.695, "state": "S (sleeping)", "threads": 106}, "restart_count": 79, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.419, "sample_count": 1098, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40314, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:25:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.33, "last_frame_pushed_at": "2026-07-01T17:25:49+09:00", "last_frame_read_age_sec": 4.609, "last_frame_read_at": "2026-07-01T17:25:49+09:00", "mjpeg": {"age_sec": 4.311, "exists": true, "mtime": "2026-07-01T17:25:49.822239+09:00", "mtime_epoch": 1782894349.8222392, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182087}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:25:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 877.273, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.267, "pushed": 3.3, "read": 3.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.015, "count": 98, "max_ms": 11.197, "p95_ms": 10.953}, "gstreamer_push": {"avg_ms": 0.94, "count": 99, "max_ms": 2.887, "p95_ms": 1.807}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.656, "count": 98, "max_ms": 71.087, "p95_ms": 67.608}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.748, "count": 98, "max_ms": 95.08, "p95_ms": 89.282}, "yolo11_640_fire_11class": {"avg_ms": 55.105, "count": 98, "max_ms": 77.283, "p95_ms": 72.365}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.926, "count": 98, "max_ms": 92.551, "p95_ms": 90.007}}, "truncated_count": 0}, "processing_total": {"avg_ms": 252.239, "count": 98, "max_ms": 327.305, "p95_ms": 277.187}, "read_interval": {"avg_ms": 261.599, "count": 98, "max_ms": 338.56, "p95_ms": 286.2}, "triton_wait": {"avg_ms": 70.368, "count": 98, "max_ms": 95.093, "p95_ms": 90.201}, "visualize": {"avg_ms": 11.712, "count": 98, "max_ms": 15.071, "p95_ms": 12.702}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 339}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3985.316, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563b01d675c0] error while decoding MB 34 38, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:27:45+09:00", "last_error_summary": "[h264 @ 0x563b01d675c0] error while decoding MB 34 38, bytestream -7", "last_exit_at": "2026-07-01T17:25:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.29, "last_started_at": "2026-07-01T17:25:52+09:00", "pid": 12156, "process": {"exists": false, "pid": 12156}, "restart_count": 79, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.285, "sample_count": 1111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40740, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:27:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T17:27:45+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-01T17:27:45+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T17:27:45.375729+09:00", "mtime_epoch": 1782894465.3757293, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190254}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:27:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 878.871, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.967, "pushed": 3.967, "read": 3.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.831, "count": 118, "max_ms": 12.491, "p95_ms": 12.104}, "gstreamer_push": {"avg_ms": 0.928, "count": 119, "max_ms": 3.015, "p95_ms": 1.179}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.993, "count": 118, "max_ms": 64.112, "p95_ms": 57.169}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.112, "count": 118, "max_ms": 91.221, "p95_ms": 86.802}, "yolo11_640_fire_11class": {"avg_ms": 51.729, "count": 118, "max_ms": 74.822, "p95_ms": 71.268}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.843, "count": 118, "max_ms": 91.744, "p95_ms": 88.084}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.973, "count": 119, "max_ms": 319.585, "p95_ms": 270.008}, "read_interval": {"avg_ms": 253.558, "count": 119, "max_ms": 332.08, "p95_ms": 280.734}, "triton_wait": {"avg_ms": 66.305, "count": 118, "max_ms": 91.757, "p95_ms": 88.835}, "visualize": {"avg_ms": 11.661, "count": 118, "max_ms": 18.386, "p95_ms": 14.697}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 339}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3984.734, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:27:45+09:00", "last_error_summary": "[h264 @ 0x563b01d675c0] error while decoding MB 34 38, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:27:45+09:00", "last_error_summary": "[h264 @ 0x563b01d675c0] error while decoding MB 34 38, bytestream -7", "last_exit_at": "2026-07-01T17:27:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.489, "last_started_at": "2026-07-01T17:25:52+09:00", "pid": 12156, "process": {"exists": false, "pid": 12156}, "restart_count": 79, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.285, "sample_count": 1111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40741, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:27:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T17:27:45+09:00", "last_frame_read_age_sec": 0.256, "last_frame_read_at": "2026-07-01T17:27:45+09:00", "mjpeg": {"age_sec": 0.246, "exists": true, "mtime": "2026-07-01T17:27:45.375729+09:00", "mtime_epoch": 1782894465.3757293, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190254}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:27:45+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 878.871, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.983, "pushed": 3.983, "read": 3.949}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.842, "count": 118, "max_ms": 12.491, "p95_ms": 12.104}, "gstreamer_push": {"avg_ms": 0.928, "count": 119, "max_ms": 3.015, "p95_ms": 1.179}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.855, "count": 118, "max_ms": 64.112, "p95_ms": 57.169}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.954, "count": 118, "max_ms": 91.221, "p95_ms": 86.802}, "yolo11_640_fire_11class": {"avg_ms": 51.567, "count": 118, "max_ms": 74.822, "p95_ms": 71.268}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.705, "count": 118, "max_ms": 91.744, "p95_ms": 88.084}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.103, "count": 119, "max_ms": 319.585, "p95_ms": 270.008}, "read_interval": {"avg_ms": 253.634, "count": 118, "max_ms": 332.08, "p95_ms": 280.734}, "triton_wait": {"avg_ms": 66.155, "count": 118, "max_ms": 91.757, "p95_ms": 88.835}, "visualize": {"avg_ms": 11.704, "count": 118, "max_ms": 18.386, "p95_ms": 14.836}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 339}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3984.656, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:27:45+09:00", "last_error_summary": "[h264 @ 0x563b01d675c0] error while decoding MB 34 38, bytestream -7", "last_exit_at": "2026-07-01T17:27:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.493, "last_started_at": "2026-07-01T17:25:52+09:00", "pid": 12156, "process": {"exists": false, "pid": 12156}, "restart_count": 79, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.285, "sample_count": 1111, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40741, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:27:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T17:27:45+09:00", "last_frame_read_age_sec": 0.26, "last_frame_read_at": "2026-07-01T17:27:45+09:00", "mjpeg": {"age_sec": 0.249, "exists": true, "mtime": "2026-07-01T17:27:45.375729+09:00", "mtime_epoch": 1782894465.3757293, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190254}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:27:45+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 878.871, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.982, "pushed": 3.982, "read": 3.949}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.842, "count": 118, "max_ms": 12.491, "p95_ms": 12.104}, "gstreamer_push": {"avg_ms": 0.928, "count": 119, "max_ms": 3.015, "p95_ms": 1.179}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.855, "count": 118, "max_ms": 64.112, "p95_ms": 57.169}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.954, "count": 118, "max_ms": 91.221, "p95_ms": 86.802}, "yolo11_640_fire_11class": {"avg_ms": 51.567, "count": 118, "max_ms": 74.822, "p95_ms": 71.268}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.705, "count": 118, "max_ms": 91.744, "p95_ms": 88.084}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.103, "count": 119, "max_ms": 319.585, "p95_ms": 270.008}, "read_interval": {"avg_ms": 253.634, "count": 118, "max_ms": 332.08, "p95_ms": 280.734}, "triton_wait": {"avg_ms": 66.155, "count": 118, "max_ms": 91.757, "p95_ms": 88.835}, "visualize": {"avg_ms": 11.704, "count": 118, "max_ms": 18.386, "p95_ms": 14.836}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 339}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4090.164, "limit_mb": 8796093022207.996}, "detail": {"pid": 12263, "restart_count": 80, "start_count": 81}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:27:45+09:00", "last_error_summary": "[h264 @ 0x563b01d675c0] error while decoding MB 34 38, bytestream -7", "last_exit_at": "2026-07-01T17:27:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:27:47+09:00", "pid": 12263, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.625, "name": "ffmpeg", "pid": 12263, "rss_mb": 136.625, "state": "S (sleeping)", "threads": 106}, "restart_count": 80, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.478, "sample_count": 1095, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40741, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:27:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.359, "last_frame_pushed_at": "2026-07-01T17:27:45+09:00", "last_frame_read_age_sec": 4.612, "last_frame_read_at": "2026-07-01T17:27:45+09:00", "mjpeg": {"age_sec": 4.341, "exists": true, "mtime": "2026-07-01T17:27:45.635733+09:00", "mtime_epoch": 1782894465.6357327, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190355}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:27:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 878.871, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.404, "pushed": 3.404, "read": 3.37}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.812, "count": 101, "max_ms": 12.491, "p95_ms": 12.104}, "gstreamer_push": {"avg_ms": 0.938, "count": 102, "max_ms": 3.015, "p95_ms": 1.179}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.321, "count": 101, "max_ms": 64.112, "p95_ms": 57.169}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.44, "count": 101, "max_ms": 91.221, "p95_ms": 86.802}, "yolo11_640_fire_11class": {"avg_ms": 52.115, "count": 101, "max_ms": 74.822, "p95_ms": 71.268}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.284, "count": 101, "max_ms": 91.744, "p95_ms": 88.084}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.888, "count": 102, "max_ms": 319.585, "p95_ms": 269.67}, "read_interval": {"avg_ms": 253.413, "count": 101, "max_ms": 332.08, "p95_ms": 279.503}, "triton_wait": {"avg_ms": 66.736, "count": 101, "max_ms": 91.757, "p95_ms": 88.835}, "visualize": {"avg_ms": 11.717, "count": 101, "max_ms": 18.386, "p95_ms": 14.836}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 343}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3987.973, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:27:45+09:00", "last_error_summary": "[h264 @ 0x563b01d675c0] error while decoding MB 34 38, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:27:45+09:00", "last_error_summary": "[h264 @ 0x563b01d675c0] error while decoding MB 34 38, bytestream -7", "last_exit_at": "2026-07-01T17:29:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.293, "last_started_at": "2026-07-01T17:27:47+09:00", "pid": 12263, "process": {"exists": false, "pid": 12263}, "restart_count": 80, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.04, "sample_count": 1112, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41163, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:29:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:29:41+09:00", "last_frame_read_age_sec": 0.258, "last_frame_read_at": "2026-07-01T17:29:41+09:00", "mjpeg": {"age_sec": 0.248, "exists": true, "mtime": "2026-07-01T17:29:41.021307+09:00", "mtime_epoch": 1782894581.0213075, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184484}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:29:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 881.324, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.825, "pushed": 3.825, "read": 3.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.279, "count": 114, "max_ms": 15.275, "p95_ms": 11.207}, "gstreamer_push": {"avg_ms": 0.894, "count": 114, "max_ms": 1.958, "p95_ms": 0.954}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.43, "count": 114, "max_ms": 71.097, "p95_ms": 66.567}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.317, "count": 114, "max_ms": 90.944, "p95_ms": 86.392}, "yolo11_640_fire_11class": {"avg_ms": 54.497, "count": 114, "max_ms": 72.702, "p95_ms": 70.433}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.012, "count": 114, "max_ms": 92.876, "p95_ms": 90.957}}, "truncated_count": 0}, "processing_total": {"avg_ms": 253.91, "count": 114, "max_ms": 317.11, "p95_ms": 275.149}, "read_interval": {"avg_ms": 263.575, "count": 113, "max_ms": 328.48, "p95_ms": 284.834}, "triton_wait": {"avg_ms": 70.546, "count": 114, "max_ms": 92.888, "p95_ms": 90.969}, "visualize": {"avg_ms": 11.898, "count": 114, "max_ms": 13.156, "p95_ms": 12.733}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.8, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 343}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3988.02, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:27:45+09:00", "last_error_summary": "[h264 @ 0x563b01d675c0] error while decoding MB 34 38, bytestream -7", "last_exit_at": "2026-07-01T17:29:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.297, "last_started_at": "2026-07-01T17:27:47+09:00", "pid": 12263, "process": {"exists": false, "pid": 12263}, "restart_count": 80, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.04, "sample_count": 1112, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41163, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:29:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:29:41+09:00", "last_frame_read_age_sec": 0.261, "last_frame_read_at": "2026-07-01T17:29:41+09:00", "mjpeg": {"age_sec": 0.252, "exists": true, "mtime": "2026-07-01T17:29:41.021307+09:00", "mtime_epoch": 1782894581.0213075, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184484}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:29:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 881.324, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.825, "pushed": 3.825, "read": 3.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.279, "count": 114, "max_ms": 15.275, "p95_ms": 11.207}, "gstreamer_push": {"avg_ms": 0.894, "count": 114, "max_ms": 1.958, "p95_ms": 0.954}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.43, "count": 114, "max_ms": 71.097, "p95_ms": 66.567}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.317, "count": 114, "max_ms": 90.944, "p95_ms": 86.392}, "yolo11_640_fire_11class": {"avg_ms": 54.497, "count": 114, "max_ms": 72.702, "p95_ms": 70.433}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.012, "count": 114, "max_ms": 92.876, "p95_ms": 90.957}}, "truncated_count": 0}, "processing_total": {"avg_ms": 253.91, "count": 114, "max_ms": 317.11, "p95_ms": 275.149}, "read_interval": {"avg_ms": 263.575, "count": 113, "max_ms": 328.48, "p95_ms": 284.834}, "triton_wait": {"avg_ms": 70.546, "count": 114, "max_ms": 92.888, "p95_ms": 90.969}, "visualize": {"avg_ms": 11.898, "count": 114, "max_ms": 13.156, "p95_ms": 12.733}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.803, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 343}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4092.867, "limit_mb": 8796093022207.996}, "detail": {"pid": 12370, "restart_count": 81, "start_count": 82}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:27:45+09:00", "last_error_summary": "[h264 @ 0x563b01d675c0] error while decoding MB 34 38, bytestream -7", "last_exit_at": "2026-07-01T17:29:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:29:44+09:00", "pid": 12370, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.492, "name": "ffmpeg", "pid": 12370, "rss_mb": 136.492, "state": "S (sleeping)", "threads": 106}, "restart_count": 81, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.055, "sample_count": 1094, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41163, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:29:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.735, "last_frame_pushed_at": "2026-07-01T17:29:41+09:00", "last_frame_read_age_sec": 4.99, "last_frame_read_at": "2026-07-01T17:29:41+09:00", "mjpeg": {"age_sec": 4.721, "exists": true, "mtime": "2026-07-01T17:29:41.281311+09:00", "mtime_epoch": 1782894581.281311, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184834}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:29:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 881.324, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.214, "pushed": 3.214, "read": 3.18}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.383, "count": 95, "max_ms": 15.275, "p95_ms": 11.268}, "gstreamer_push": {"avg_ms": 0.898, "count": 96, "max_ms": 1.958, "p95_ms": 0.963}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.863, "count": 95, "max_ms": 71.097, "p95_ms": 66.567}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.931, "count": 95, "max_ms": 90.944, "p95_ms": 86.392}, "yolo11_640_fire_11class": {"avg_ms": 54.762, "count": 95, "max_ms": 72.702, "p95_ms": 70.433}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.552, "count": 95, "max_ms": 92.876, "p95_ms": 91.013}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.057, "count": 96, "max_ms": 317.11, "p95_ms": 275.855}, "read_interval": {"avg_ms": 264.74, "count": 95, "max_ms": 328.48, "p95_ms": 285.314}, "triton_wait": {"avg_ms": 71.174, "count": 95, "max_ms": 92.888, "p95_ms": 91.026}, "visualize": {"avg_ms": 11.906, "count": 95, "max_ms": 13.156, "p95_ms": 12.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 346}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3981.547, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56469b672c00] error while decoding MB 52 40, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:31:32+09:00", "last_error_summary": "[h264 @ 0x56469b672c00] error while decoding MB 52 40, bytestream -5", "last_exit_at": "2026-07-01T17:29:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.689, "last_started_at": "2026-07-01T17:29:44+09:00", "pid": 12370, "process": {"exists": false, "pid": 12370}, "restart_count": 81, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.922, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41601, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:31:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.045, "last_frame_pushed_at": "2026-07-01T17:31:32+09:00", "last_frame_read_age_sec": 0.037, "last_frame_read_at": "2026-07-01T17:31:32+09:00", "mjpeg": {"age_sec": 0.027, "exists": true, "mtime": "2026-07-01T17:31:32.662897+09:00", "mtime_epoch": 1782894692.662897, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:31:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 870.516, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.281, "pushed": 4.281, "read": 4.281}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.837, "count": 127, "max_ms": 10.413, "p95_ms": 10.188}, "gstreamer_push": {"avg_ms": 0.9, "count": 128, "max_ms": 2.003, "p95_ms": 1.007}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.594, "count": 127, "max_ms": 70.993, "p95_ms": 66.033}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.255, "count": 127, "max_ms": 89.817, "p95_ms": 86.798}, "yolo11_640_fire_11class": {"avg_ms": 54.712, "count": 127, "max_ms": 75.345, "p95_ms": 70.162}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.899, "count": 127, "max_ms": 92.598, "p95_ms": 89.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.547, "count": 128, "max_ms": 283.573, "p95_ms": 254.499}, "read_interval": {"avg_ms": 235.185, "count": 128, "max_ms": 293.872, "p95_ms": 264.954}, "triton_wait": {"avg_ms": 69.113, "count": 127, "max_ms": 92.611, "p95_ms": 89.097}, "visualize": {"avg_ms": 10.48, "count": 127, "max_ms": 15.083, "p95_ms": 11.66}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 346}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3983.645, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:31:32+09:00", "last_error_summary": "[h264 @ 0x56469b672c00] error while decoding MB 52 40, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:31:32+09:00", "last_error_summary": "[h264 @ 0x56469b672c00] error while decoding MB 52 40, bytestream -5", "last_exit_at": "2026-07-01T17:31:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.957, "last_started_at": "2026-07-01T17:29:44+09:00", "pid": 12370, "process": {"exists": false, "pid": 12370}, "restart_count": 81, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.097, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41602, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:31:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T17:31:32+09:00", "last_frame_read_age_sec": 0.306, "last_frame_read_at": "2026-07-01T17:31:32+09:00", "mjpeg": {"age_sec": 0.295, "exists": true, "mtime": "2026-07-01T17:31:32.662897+09:00", "mtime_epoch": 1782894692.662897, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:31:32+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 877.641, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.278, "pushed": 4.278, "read": 4.245}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.828, "count": 127, "max_ms": 10.413, "p95_ms": 10.188}, "gstreamer_push": {"avg_ms": 0.917, "count": 128, "max_ms": 3.003, "p95_ms": 1.016}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.711, "count": 127, "max_ms": 70.993, "p95_ms": 66.033}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.328, "count": 127, "max_ms": 89.817, "p95_ms": 86.798}, "yolo11_640_fire_11class": {"avg_ms": 54.791, "count": 127, "max_ms": 75.345, "p95_ms": 70.162}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.98, "count": 127, "max_ms": 92.598, "p95_ms": 89.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.002, "count": 128, "max_ms": 298.188, "p95_ms": 262.385}, "read_interval": {"avg_ms": 235.072, "count": 127, "max_ms": 293.872, "p95_ms": 264.954}, "triton_wait": {"avg_ms": 69.194, "count": 127, "max_ms": 92.611, "p95_ms": 89.097}, "visualize": {"avg_ms": 10.472, "count": 127, "max_ms": 15.083, "p95_ms": 11.626}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 346}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3983.637, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:31:32+09:00", "last_error_summary": "[h264 @ 0x56469b672c00] error while decoding MB 52 40, bytestream -5", "last_exit_at": "2026-07-01T17:31:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.96, "last_started_at": "2026-07-01T17:29:44+09:00", "pid": 12370, "process": {"exists": false, "pid": 12370}, "restart_count": 81, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.097, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41602, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:31:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T17:31:32+09:00", "last_frame_read_age_sec": 0.309, "last_frame_read_at": "2026-07-01T17:31:32+09:00", "mjpeg": {"age_sec": 0.298, "exists": true, "mtime": "2026-07-01T17:31:32.662897+09:00", "mtime_epoch": 1782894692.662897, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:31:32+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 877.641, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.278, "pushed": 4.278, "read": 4.244}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.828, "count": 127, "max_ms": 10.413, "p95_ms": 10.188}, "gstreamer_push": {"avg_ms": 0.917, "count": 128, "max_ms": 3.003, "p95_ms": 1.016}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.711, "count": 127, "max_ms": 70.993, "p95_ms": 66.033}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.328, "count": 127, "max_ms": 89.817, "p95_ms": 86.798}, "yolo11_640_fire_11class": {"avg_ms": 54.791, "count": 127, "max_ms": 75.345, "p95_ms": 70.162}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.98, "count": 127, "max_ms": 92.598, "p95_ms": 89.087}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.002, "count": 128, "max_ms": 298.188, "p95_ms": 262.385}, "read_interval": {"avg_ms": 235.072, "count": 127, "max_ms": 293.872, "p95_ms": 264.954}, "triton_wait": {"avg_ms": 69.194, "count": 127, "max_ms": 92.611, "p95_ms": 89.097}, "visualize": {"avg_ms": 10.472, "count": 127, "max_ms": 15.083, "p95_ms": 11.626}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 346}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4091.527, "limit_mb": 8796093022207.996}, "detail": {"pid": 12477, "restart_count": 82, "start_count": 83}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:31:32+09:00", "last_error_summary": "[h264 @ 0x56469b672c00] error while decoding MB 52 40, bytestream -5", "last_exit_at": "2026-07-01T17:31:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:31:35+09:00", "pid": 12477, "process": {"exists": true, "fd_count": 5, "hwm_mb": 134.027, "name": "ffmpeg", "pid": 12477, "rss_mb": 134.027, "state": "S (sleeping)", "threads": 106}, "restart_count": 82, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.922, "sample_count": 1125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41602, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:31:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.627, "last_frame_pushed_at": "2026-07-01T17:31:32+09:00", "last_frame_read_age_sec": 4.929, "last_frame_read_at": "2026-07-01T17:31:32+09:00", "mjpeg": {"age_sec": 4.611, "exists": true, "mtime": "2026-07-01T17:31:32.970902+09:00", "mtime_epoch": 1782894692.9709017, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200006}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:31:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 877.641, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.654, "pushed": 3.654, "read": 3.62}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.682, "count": 109, "max_ms": 10.413, "p95_ms": 9.546}, "gstreamer_push": {"avg_ms": 0.913, "count": 109, "max_ms": 3.003, "p95_ms": 0.973}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.343, "count": 109, "max_ms": 68.57, "p95_ms": 64.843}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.567, "count": 109, "max_ms": 88.172, "p95_ms": 83.985}, "yolo11_640_fire_11class": {"avg_ms": 54.274, "count": 109, "max_ms": 71.747, "p95_ms": 69.164}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.311, "count": 109, "max_ms": 89.147, "p95_ms": 87.71}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.524, "count": 109, "max_ms": 298.188, "p95_ms": 248.007}, "read_interval": {"avg_ms": 232.438, "count": 108, "max_ms": 293.872, "p95_ms": 256.549}, "triton_wait": {"avg_ms": 68.46, "count": 109, "max_ms": 89.16, "p95_ms": 87.729}, "visualize": {"avg_ms": 10.394, "count": 109, "max_ms": 15.083, "p95_ms": 11.204}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.832, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 350}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3986.961, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562105410040] error while decoding MB 10 31, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:33:27+09:00", "last_error_summary": "[h264 @ 0x562105410040] error while decoding MB 10 31, bytestream -5", "last_exit_at": "2026-07-01T17:31:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.971, "last_started_at": "2026-07-01T17:31:35+09:00", "pid": 12477, "process": {"exists": false, "pid": 12477}, "restart_count": 82, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.149, "sample_count": 1138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42021, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:33:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T17:33:27+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T17:33:27+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T17:33:27.504587+09:00", "mtime_epoch": 1782894807.5045867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194319}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:33:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 880.531, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.735, "pushed": 3.735, "read": 3.735}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.274, "count": 111, "max_ms": 12.996, "p95_ms": 11.297}, "gstreamer_push": {"avg_ms": 0.944, "count": 112, "max_ms": 2.94, "p95_ms": 1.811}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.664, "count": 111, "max_ms": 80.434, "p95_ms": 69.945}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.679, "count": 111, "max_ms": 97.831, "p95_ms": 90.225}, "yolo11_640_fire_11class": {"avg_ms": 57.214, "count": 111, "max_ms": 80.422, "p95_ms": 71.564}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.776, "count": 111, "max_ms": 97.064, "p95_ms": 91.382}}, "truncated_count": 0}, "processing_total": {"avg_ms": 259.772, "count": 112, "max_ms": 341.272, "p95_ms": 282.048}, "read_interval": {"avg_ms": 269.378, "count": 112, "max_ms": 351.058, "p95_ms": 291.548}, "triton_wait": {"avg_ms": 74.303, "count": 111, "max_ms": 97.843, "p95_ms": 91.431}, "visualize": {"avg_ms": 12.022, "count": 111, "max_ms": 17.784, "p95_ms": 13.103}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 350}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3989.84, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:33:27+09:00", "last_error_summary": "[h264 @ 0x562105410040] error while decoding MB 10 31, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:33:27+09:00", "last_error_summary": "[h264 @ 0x562105410040] error while decoding MB 10 31, bytestream -5", "last_exit_at": "2026-07-01T17:33:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.192, "last_started_at": "2026-07-01T17:31:35+09:00", "pid": 12477, "process": {"exists": false, "pid": 12477}, "restart_count": 82, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.149, "sample_count": 1138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42022, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:33:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T17:33:27+09:00", "last_frame_read_age_sec": 0.274, "last_frame_read_at": "2026-07-01T17:33:27+09:00", "mjpeg": {"age_sec": 0.268, "exists": true, "mtime": "2026-07-01T17:33:27.504587+09:00", "mtime_epoch": 1782894807.5045867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194319}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:33:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 884.141, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.738, "pushed": 3.738, "read": 3.705}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.273, "count": 111, "max_ms": 12.996, "p95_ms": 11.297}, "gstreamer_push": {"avg_ms": 0.943, "count": 112, "max_ms": 2.94, "p95_ms": 1.811}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.832, "count": 111, "max_ms": 80.434, "p95_ms": 69.945}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.924, "count": 111, "max_ms": 97.831, "p95_ms": 90.225}, "yolo11_640_fire_11class": {"avg_ms": 57.435, "count": 111, "max_ms": 80.422, "p95_ms": 71.564}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.035, "count": 111, "max_ms": 97.064, "p95_ms": 91.382}}, "truncated_count": 0}, "processing_total": {"avg_ms": 260.057, "count": 112, "max_ms": 341.272, "p95_ms": 282.048}, "read_interval": {"avg_ms": 269.615, "count": 111, "max_ms": 351.058, "p95_ms": 291.548}, "triton_wait": {"avg_ms": 74.555, "count": 111, "max_ms": 97.843, "p95_ms": 91.431}, "visualize": {"avg_ms": 12.024, "count": 111, "max_ms": 17.784, "p95_ms": 13.103}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 350}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3989.832, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:33:27+09:00", "last_error_summary": "[h264 @ 0x562105410040] error while decoding MB 10 31, bytestream -5", "last_exit_at": "2026-07-01T17:33:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.195, "last_started_at": "2026-07-01T17:31:35+09:00", "pid": 12477, "process": {"exists": false, "pid": 12477}, "restart_count": 82, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.149, "sample_count": 1138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42022, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:33:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T17:33:27+09:00", "last_frame_read_age_sec": 0.277, "last_frame_read_at": "2026-07-01T17:33:27+09:00", "mjpeg": {"age_sec": 0.271, "exists": true, "mtime": "2026-07-01T17:33:27.504587+09:00", "mtime_epoch": 1782894807.5045867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194319}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:33:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 884.141, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.738, "pushed": 3.738, "read": 3.705}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.273, "count": 111, "max_ms": 12.996, "p95_ms": 11.297}, "gstreamer_push": {"avg_ms": 0.943, "count": 112, "max_ms": 2.94, "p95_ms": 1.811}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.832, "count": 111, "max_ms": 80.434, "p95_ms": 69.945}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.924, "count": 111, "max_ms": 97.831, "p95_ms": 90.225}, "yolo11_640_fire_11class": {"avg_ms": 57.435, "count": 111, "max_ms": 80.422, "p95_ms": 71.564}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.035, "count": 111, "max_ms": 97.064, "p95_ms": 91.382}}, "truncated_count": 0}, "processing_total": {"avg_ms": 260.057, "count": 112, "max_ms": 341.272, "p95_ms": 282.048}, "read_interval": {"avg_ms": 269.615, "count": 111, "max_ms": 351.058, "p95_ms": 291.548}, "triton_wait": {"avg_ms": 74.555, "count": 111, "max_ms": 97.843, "p95_ms": 91.431}, "visualize": {"avg_ms": 12.024, "count": 111, "max_ms": 17.784, "p95_ms": 13.103}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 350}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4098.16, "limit_mb": 8796093022207.996}, "detail": {"pid": 12584, "restart_count": 83, "start_count": 84}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:33:27+09:00", "last_error_summary": "[h264 @ 0x562105410040] error while decoding MB 10 31, bytestream -5", "last_exit_at": "2026-07-01T17:33:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:33:30+09:00", "pid": 12584, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.199, "name": "ffmpeg", "pid": 12584, "rss_mb": 136.199, "state": "S (sleeping)", "threads": 106}, "restart_count": 83, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.149, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42022, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:33:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.531, "last_frame_pushed_at": "2026-07-01T17:33:27+09:00", "last_frame_read_age_sec": 4.801, "last_frame_read_at": "2026-07-01T17:33:27+09:00", "mjpeg": {"age_sec": 4.515, "exists": true, "mtime": "2026-07-01T17:33:27.784591+09:00", "mtime_epoch": 1782894807.7845907, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:33:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 884.141, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.172, "pushed": 3.172, "read": 3.139}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.31, "count": 94, "max_ms": 12.996, "p95_ms": 11.454}, "gstreamer_push": {"avg_ms": 0.936, "count": 95, "max_ms": 2.896, "p95_ms": 1.811}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.208, "count": 94, "max_ms": 80.434, "p95_ms": 69.992}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.337, "count": 94, "max_ms": 97.831, "p95_ms": 90.225}, "yolo11_640_fire_11class": {"avg_ms": 57.811, "count": 94, "max_ms": 80.422, "p95_ms": 72.29}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.421, "count": 94, "max_ms": 97.064, "p95_ms": 91.522}}, "truncated_count": 0}, "processing_total": {"avg_ms": 260.437, "count": 95, "max_ms": 341.272, "p95_ms": 282.048}, "read_interval": {"avg_ms": 270.13, "count": 94, "max_ms": 351.058, "p95_ms": 291.548}, "triton_wait": {"avg_ms": 74.965, "count": 94, "max_ms": 97.843, "p95_ms": 91.535}, "visualize": {"avg_ms": 12.066, "count": 94, "max_ms": 17.784, "p95_ms": 13.271}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 353}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3982.629, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:33:27+09:00", "last_error_summary": "[h264 @ 0x562105410040] error while decoding MB 10 31, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:33:27+09:00", "last_error_summary": "[h264 @ 0x562105410040] error while decoding MB 10 31, bytestream -5", "last_exit_at": "2026-07-01T17:35:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.398, "last_started_at": "2026-07-01T17:33:30+09:00", "pid": 12584, "process": {"exists": false, "pid": 12584}, "restart_count": 83, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.446, "sample_count": 1138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42450, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:35:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T17:35:22+09:00", "last_frame_read_age_sec": 0.244, "last_frame_read_at": "2026-07-01T17:35:22+09:00", "mjpeg": {"age_sec": 0.234, "exists": true, "mtime": "2026-07-01T17:35:22.462322+09:00", "mtime_epoch": 1782894922.4623218, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195886}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:35:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 876.402, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.92, "pushed": 3.92, "read": 3.886}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.949, "count": 116, "max_ms": 12.829, "p95_ms": 11.162}, "gstreamer_push": {"avg_ms": 0.916, "count": 117, "max_ms": 2.942, "p95_ms": 0.935}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.576, "count": 116, "max_ms": 71.309, "p95_ms": 61.7}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.705, "count": 116, "max_ms": 92.325, "p95_ms": 90.444}, "yolo11_640_fire_11class": {"avg_ms": 52.751, "count": 116, "max_ms": 70.478, "p95_ms": 68.333}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.356, "count": 116, "max_ms": 93.482, "p95_ms": 90.708}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.548, "count": 117, "max_ms": 320.048, "p95_ms": 274.711}, "read_interval": {"avg_ms": 257.283, "count": 116, "max_ms": 331.337, "p95_ms": 285.123}, "triton_wait": {"avg_ms": 68.82, "count": 116, "max_ms": 93.494, "p95_ms": 90.751}, "visualize": {"avg_ms": 11.667, "count": 116, "max_ms": 14.694, "p95_ms": 13.042}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.848, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 353}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3982.621, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:33:27+09:00", "last_error_summary": "[h264 @ 0x562105410040] error while decoding MB 10 31, bytestream -5", "last_exit_at": "2026-07-01T17:35:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.401, "last_started_at": "2026-07-01T17:33:30+09:00", "pid": 12584, "process": {"exists": false, "pid": 12584}, "restart_count": 83, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.446, "sample_count": 1138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42450, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:35:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T17:35:22+09:00", "last_frame_read_age_sec": 0.247, "last_frame_read_at": "2026-07-01T17:35:22+09:00", "mjpeg": {"age_sec": 0.237, "exists": true, "mtime": "2026-07-01T17:35:22.462322+09:00", "mtime_epoch": 1782894922.4623218, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195886}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:35:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 876.402, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.919, "pushed": 3.919, "read": 3.886}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.949, "count": 116, "max_ms": 12.829, "p95_ms": 11.162}, "gstreamer_push": {"avg_ms": 0.916, "count": 117, "max_ms": 2.942, "p95_ms": 0.935}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.576, "count": 116, "max_ms": 71.309, "p95_ms": 61.7}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.705, "count": 116, "max_ms": 92.325, "p95_ms": 90.444}, "yolo11_640_fire_11class": {"avg_ms": 52.751, "count": 116, "max_ms": 70.478, "p95_ms": 68.333}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.356, "count": 116, "max_ms": 93.482, "p95_ms": 90.708}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.548, "count": 117, "max_ms": 320.048, "p95_ms": 274.711}, "read_interval": {"avg_ms": 257.283, "count": 116, "max_ms": 331.337, "p95_ms": 285.123}, "triton_wait": {"avg_ms": 68.82, "count": 116, "max_ms": 93.494, "p95_ms": 90.751}, "visualize": {"avg_ms": 11.667, "count": 116, "max_ms": 14.694, "p95_ms": 13.042}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.851, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 353}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4090.164, "limit_mb": 8796093022207.996}, "detail": {"pid": 12691, "restart_count": 84, "start_count": 85}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:33:27+09:00", "last_error_summary": "[h264 @ 0x562105410040] error while decoding MB 10 31, bytestream -5", "last_exit_at": "2026-07-01T17:35:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:35:25+09:00", "pid": 12691, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.496, "name": "ffmpeg", "pid": 12691, "rss_mb": 136.496, "state": "S (sleeping)", "threads": 106}, "restart_count": 84, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.507, "sample_count": 1120, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42450, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:35:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.611, "last_frame_pushed_at": "2026-07-01T17:35:22+09:00", "last_frame_read_age_sec": 4.85, "last_frame_read_at": "2026-07-01T17:35:22+09:00", "mjpeg": {"age_sec": 4.593, "exists": true, "mtime": "2026-07-01T17:35:22.710326+09:00", "mtime_epoch": 1782894922.7103257, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:35:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 876.402, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.335, "pushed": 3.335, "read": 3.302}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.91, "count": 99, "max_ms": 12.829, "p95_ms": 11.399}, "gstreamer_push": {"avg_ms": 0.928, "count": 100, "max_ms": 2.942, "p95_ms": 0.935}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.044, "count": 99, "max_ms": 71.309, "p95_ms": 61.056}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.881, "count": 99, "max_ms": 90.739, "p95_ms": 87.73}, "yolo11_640_fire_11class": {"avg_ms": 51.448, "count": 99, "max_ms": 70.426, "p95_ms": 67.752}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.415, "count": 99, "max_ms": 90.708, "p95_ms": 87.284}}, "truncated_count": 0}, "processing_total": {"avg_ms": 246.876, "count": 100, "max_ms": 320.048, "p95_ms": 272.05}, "read_interval": {"avg_ms": 256.626, "count": 99, "max_ms": 331.337, "p95_ms": 282.851}, "triton_wait": {"avg_ms": 66.918, "count": 99, "max_ms": 90.751, "p95_ms": 88.579}, "visualize": {"avg_ms": 11.706, "count": 99, "max_ms": 14.694, "p95_ms": 13.065}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 357}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3994.945, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563dcedb63c0] error while decoding MB 42 5, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:37:18+09:00", "last_error_summary": "[h264 @ 0x563dcedb63c0] error while decoding MB 42 5, bytestream -13", "last_exit_at": "2026-07-01T17:35:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.768, "last_started_at": "2026-07-01T17:35:25+09:00", "pid": 12691, "process": {"exists": false, "pid": 12691}, "restart_count": 84, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.948, "sample_count": 1121, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42878, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:37:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T17:37:18+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T17:37:18+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T17:37:18.028102+09:00", "mtime_epoch": 1782895038.0281017, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 214277}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:37:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 884.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.897, "pushed": 3.897, "read": 3.897}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.049, "count": 116, "max_ms": 11.42, "p95_ms": 10.963}, "gstreamer_push": {"avg_ms": 0.927, "count": 116, "max_ms": 2.889, "p95_ms": 1.741}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.343, "count": 116, "max_ms": 70.471, "p95_ms": 58.165}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.752, "count": 116, "max_ms": 86.178, "p95_ms": 83.438}, "yolo11_640_fire_11class": {"avg_ms": 51.045, "count": 116, "max_ms": 73.589, "p95_ms": 70.993}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.228, "count": 116, "max_ms": 92.729, "p95_ms": 88.548}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.351, "count": 116, "max_ms": 349.029, "p95_ms": 271.899}, "read_interval": {"avg_ms": 258.311, "count": 116, "max_ms": 360.314, "p95_ms": 281.323}, "triton_wait": {"avg_ms": 65.589, "count": 116, "max_ms": 92.741, "p95_ms": 88.56}, "visualize": {"avg_ms": 11.778, "count": 116, "max_ms": 16.056, "p95_ms": 12.317}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.769, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 357}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3994.18, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:37:18+09:00", "last_error_summary": "[h264 @ 0x563dcedb63c0] error while decoding MB 42 5, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:37:18+09:00", "last_error_summary": "[h264 @ 0x563dcedb63c0] error while decoding MB 42 5, bytestream -13", "last_exit_at": "2026-07-01T17:37:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.958, "last_started_at": "2026-07-01T17:35:25+09:00", "pid": 12691, "process": {"exists": false, "pid": 12691}, "restart_count": 84, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.948, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42879, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:37:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:37:18+09:00", "last_frame_read_age_sec": 0.239, "last_frame_read_at": "2026-07-01T17:37:18+09:00", "mjpeg": {"age_sec": 0.232, "exists": true, "mtime": "2026-07-01T17:37:18.028102+09:00", "mtime_epoch": 1782895038.0281017, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 214277}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:37:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 884.672, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.905, "pushed": 3.905, "read": 3.872}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.053, "count": 116, "max_ms": 11.42, "p95_ms": 10.963}, "gstreamer_push": {"avg_ms": 0.926, "count": 117, "max_ms": 2.889, "p95_ms": 1.741}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.35, "count": 116, "max_ms": 70.471, "p95_ms": 58.165}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.727, "count": 116, "max_ms": 86.178, "p95_ms": 83.438}, "yolo11_640_fire_11class": {"avg_ms": 51.041, "count": 116, "max_ms": 73.589, "p95_ms": 70.993}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.215, "count": 116, "max_ms": 92.729, "p95_ms": 88.548}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.232, "count": 117, "max_ms": 349.029, "p95_ms": 271.899}, "read_interval": {"avg_ms": 258.311, "count": 116, "max_ms": 360.314, "p95_ms": 281.323}, "triton_wait": {"avg_ms": 65.576, "count": 116, "max_ms": 92.741, "p95_ms": 88.56}, "visualize": {"avg_ms": 11.77, "count": 116, "max_ms": 16.056, "p95_ms": 12.317}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 357}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3994.172, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:37:18+09:00", "last_error_summary": "[h264 @ 0x563dcedb63c0] error while decoding MB 42 5, bytestream -13", "last_exit_at": "2026-07-01T17:37:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.961, "last_started_at": "2026-07-01T17:35:25+09:00", "pid": 12691, "process": {"exists": false, "pid": 12691}, "restart_count": 84, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.948, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42879, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:37:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:37:18+09:00", "last_frame_read_age_sec": 0.242, "last_frame_read_at": "2026-07-01T17:37:18+09:00", "mjpeg": {"age_sec": 0.236, "exists": true, "mtime": "2026-07-01T17:37:18.028102+09:00", "mtime_epoch": 1782895038.0281017, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 214277}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:37:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 884.672, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.905, "pushed": 3.905, "read": 3.872}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.053, "count": 116, "max_ms": 11.42, "p95_ms": 10.963}, "gstreamer_push": {"avg_ms": 0.926, "count": 117, "max_ms": 2.889, "p95_ms": 1.741}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.35, "count": 116, "max_ms": 70.471, "p95_ms": 58.165}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.727, "count": 116, "max_ms": 86.178, "p95_ms": 83.438}, "yolo11_640_fire_11class": {"avg_ms": 51.041, "count": 116, "max_ms": 73.589, "p95_ms": 70.993}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.215, "count": 116, "max_ms": 92.729, "p95_ms": 88.548}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.232, "count": 117, "max_ms": 349.029, "p95_ms": 271.899}, "read_interval": {"avg_ms": 258.311, "count": 116, "max_ms": 360.314, "p95_ms": 281.323}, "triton_wait": {"avg_ms": 65.576, "count": 116, "max_ms": 92.741, "p95_ms": 88.56}, "visualize": {"avg_ms": 11.77, "count": 116, "max_ms": 16.056, "p95_ms": 12.317}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 357}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4096.949, "limit_mb": 8796093022207.996}, "detail": {"pid": 12798, "restart_count": 85, "start_count": 86}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:37:18+09:00", "last_error_summary": "[h264 @ 0x563dcedb63c0] error while decoding MB 42 5, bytestream -13", "last_exit_at": "2026-07-01T17:37:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:37:20+09:00", "pid": 12798, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.203, "name": "ffmpeg", "pid": 12798, "rss_mb": 139.203, "state": "S (sleeping)", "threads": 106}, "restart_count": 85, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.948, "sample_count": 1103, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 42879, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:37:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.681, "last_frame_pushed_at": "2026-07-01T17:37:18+09:00", "last_frame_read_age_sec": 4.917, "last_frame_read_at": "2026-07-01T17:37:18+09:00", "mjpeg": {"age_sec": 4.663, "exists": true, "mtime": "2026-07-01T17:37:18.276106+09:00", "mtime_epoch": 1782895038.2761056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 214193}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:37:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 884.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.285, "pushed": 3.285, "read": 3.251}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.083, "count": 98, "max_ms": 11.42, "p95_ms": 10.997}, "gstreamer_push": {"avg_ms": 0.931, "count": 98, "max_ms": 2.889, "p95_ms": 1.741}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.866, "count": 98, "max_ms": 70.471, "p95_ms": 58.276}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.332, "count": 98, "max_ms": 86.178, "p95_ms": 83.467}, "yolo11_640_fire_11class": {"avg_ms": 51.825, "count": 98, "max_ms": 73.589, "p95_ms": 71.598}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.06, "count": 98, "max_ms": 92.729, "p95_ms": 89.074}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.142, "count": 98, "max_ms": 349.029, "p95_ms": 273.858}, "read_interval": {"avg_ms": 259.306, "count": 97, "max_ms": 360.314, "p95_ms": 282.898}, "triton_wait": {"avg_ms": 66.419, "count": 98, "max_ms": 92.741, "p95_ms": 89.086}, "visualize": {"avg_ms": 11.723, "count": 98, "max_ms": 14.196, "p95_ms": 12.317}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.833, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 360}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3996.469, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558892755180] error while decoding MB 73 44, bytestream -22", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:39:10+09:00", "last_error_summary": "[h264 @ 0x558892755180] error while decoding MB 73 44, bytestream -22", "last_exit_at": "2026-07-01T17:37:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.016, "last_started_at": "2026-07-01T17:37:20+09:00", "pid": 12798, "process": {"exists": false, "pid": 12798}, "restart_count": 85, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.156, "sample_count": 1133, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43303, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:39:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T17:39:10+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T17:39:10+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T17:39:10.909868+09:00", "mtime_epoch": 1782895150.9098678, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 214604}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:39:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 885.238, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.945, "pushed": 3.945, "read": 3.945}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.862, "count": 117, "max_ms": 13.032, "p95_ms": 10.901}, "gstreamer_push": {"avg_ms": 0.896, "count": 118, "max_ms": 1.921, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.747, "count": 117, "max_ms": 68.641, "p95_ms": 59.031}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.462, "count": 117, "max_ms": 96.24, "p95_ms": 89.144}, "yolo11_640_fire_11class": {"avg_ms": 52.437, "count": 117, "max_ms": 72.86, "p95_ms": 70.309}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.617, "count": 117, "max_ms": 93.623, "p95_ms": 88.148}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.621, "count": 118, "max_ms": 339.02, "p95_ms": 271.762}, "read_interval": {"avg_ms": 255.112, "count": 118, "max_ms": 348.449, "p95_ms": 279.46}, "triton_wait": {"avg_ms": 67.307, "count": 117, "max_ms": 96.252, "p95_ms": 89.233}, "visualize": {"avg_ms": 11.501, "count": 117, "max_ms": 14.918, "p95_ms": 12.713}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 360}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3995.238, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:39:10+09:00", "last_error_summary": "[h264 @ 0x558892755180] error while decoding MB 73 44, bytestream -22", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:39:10+09:00", "last_error_summary": "[h264 @ 0x558892755180] error while decoding MB 73 44, bytestream -22", "last_exit_at": "2026-07-01T17:39:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.212, "last_started_at": "2026-07-01T17:37:20+09:00", "pid": 12798, "process": {"exists": false, "pid": 12798}, "restart_count": 85, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.156, "sample_count": 1133, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:39:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:39:11+09:00", "last_frame_read_age_sec": 0.247, "last_frame_read_at": "2026-07-01T17:39:10+09:00", "mjpeg": {"age_sec": 0.24, "exists": true, "mtime": "2026-07-01T17:39:10.909868+09:00", "mtime_epoch": 1782895150.9098678, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 214604}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:39:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 885.238, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.953, "pushed": 3.953, "read": 3.919}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.868, "count": 117, "max_ms": 13.032, "p95_ms": 11.057}, "gstreamer_push": {"avg_ms": 0.896, "count": 118, "max_ms": 1.921, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.729, "count": 117, "max_ms": 68.641, "p95_ms": 59.031}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.458, "count": 117, "max_ms": 96.24, "p95_ms": 89.144}, "yolo11_640_fire_11class": {"avg_ms": 52.452, "count": 117, "max_ms": 72.86, "p95_ms": 70.309}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.599, "count": 117, "max_ms": 93.623, "p95_ms": 88.148}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.595, "count": 118, "max_ms": 339.02, "p95_ms": 271.762}, "read_interval": {"avg_ms": 255.112, "count": 117, "max_ms": 348.449, "p95_ms": 279.46}, "triton_wait": {"avg_ms": 67.289, "count": 117, "max_ms": 96.252, "p95_ms": 89.233}, "visualize": {"avg_ms": 11.502, "count": 117, "max_ms": 14.918, "p95_ms": 12.713}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.852, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 360}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3995.23, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:39:10+09:00", "last_error_summary": "[h264 @ 0x558892755180] error while decoding MB 73 44, bytestream -22", "last_exit_at": "2026-07-01T17:39:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.215, "last_started_at": "2026-07-01T17:37:20+09:00", "pid": 12798, "process": {"exists": false, "pid": 12798}, "restart_count": 85, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.156, "sample_count": 1133, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:39:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:39:11+09:00", "last_frame_read_age_sec": 0.25, "last_frame_read_at": "2026-07-01T17:39:10+09:00", "mjpeg": {"age_sec": 0.243, "exists": true, "mtime": "2026-07-01T17:39:10.909868+09:00", "mtime_epoch": 1782895150.9098678, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 214604}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:39:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 885.238, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.952, "pushed": 3.952, "read": 3.919}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.868, "count": 117, "max_ms": 13.032, "p95_ms": 11.057}, "gstreamer_push": {"avg_ms": 0.896, "count": 118, "max_ms": 1.921, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.729, "count": 117, "max_ms": 68.641, "p95_ms": 59.031}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.458, "count": 117, "max_ms": 96.24, "p95_ms": 89.144}, "yolo11_640_fire_11class": {"avg_ms": 52.452, "count": 117, "max_ms": 72.86, "p95_ms": 70.309}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.599, "count": 117, "max_ms": 93.623, "p95_ms": 88.148}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.595, "count": 118, "max_ms": 339.02, "p95_ms": 271.762}, "read_interval": {"avg_ms": 255.112, "count": 117, "max_ms": 348.449, "p95_ms": 279.46}, "triton_wait": {"avg_ms": 67.289, "count": 117, "max_ms": 96.252, "p95_ms": 89.233}, "visualize": {"avg_ms": 11.502, "count": 117, "max_ms": 14.918, "p95_ms": 12.713}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.855, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 360}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4101.039, "limit_mb": 8796093022207.996}, "detail": {"pid": 12905, "restart_count": 86, "start_count": 87}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:39:10+09:00", "last_error_summary": "[h264 @ 0x558892755180] error while decoding MB 73 44, bytestream -22", "last_exit_at": "2026-07-01T17:39:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:39:14+09:00", "pid": 12905, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.453, "name": "ffmpeg", "pid": 12905, "rss_mb": 136.453, "state": "S (sleeping)", "threads": 106}, "restart_count": 86, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.543, "sample_count": 1113, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43304, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:39:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.958, "last_frame_pushed_at": "2026-07-01T17:39:11+09:00", "last_frame_read_age_sec": 5.202, "last_frame_read_at": "2026-07-01T17:39:10+09:00", "mjpeg": {"age_sec": 4.943, "exists": true, "mtime": "2026-07-01T17:39:11.161872+09:00", "mtime_epoch": 1782895151.1618717, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 214771}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:39:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 886.25, "name": "celery", "pid": 237, "rss_mb": 885.238, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.309, "pushed": 3.309, "read": 3.276}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.81, "count": 98, "max_ms": 13.032, "p95_ms": 11.172}, "gstreamer_push": {"avg_ms": 0.905, "count": 99, "max_ms": 1.921, "p95_ms": 0.954}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.189, "count": 98, "max_ms": 68.641, "p95_ms": 60.337}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.13, "count": 98, "max_ms": 90.961, "p95_ms": 87.493}, "yolo11_640_fire_11class": {"avg_ms": 52.339, "count": 98, "max_ms": 71.287, "p95_ms": 69.19}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.525, "count": 98, "max_ms": 93.623, "p95_ms": 88.044}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.136, "count": 99, "max_ms": 339.02, "p95_ms": 271.762}, "read_interval": {"avg_ms": 254.663, "count": 98, "max_ms": 348.449, "p95_ms": 279.46}, "triton_wait": {"avg_ms": 67.062, "count": 98, "max_ms": 93.635, "p95_ms": 88.056}, "visualize": {"avg_ms": 11.432, "count": 98, "max_ms": 13.1, "p95_ms": 12.713}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 364}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3988.52, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f009a29b00] error while decoding MB 51 43, bytestream -32", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:41:05+09:00", "last_error_summary": "[h264 @ 0x55f009a29b00] error while decoding MB 51 43, bytestream -32", "last_exit_at": "2026-07-01T17:39:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.934, "last_started_at": "2026-07-01T17:39:14+09:00", "pid": 12905, "process": {"exists": false, "pid": 12905}, "restart_count": 86, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.948, "sample_count": 1119, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43716, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:41:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T17:41:04+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T17:41:04+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T17:41:04.999675+09:00", "mtime_epoch": 1782895264.9996748, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203261}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:41:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 888.09, "name": "celery", "pid": 237, "rss_mb": 881.977, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.71, "pushed": 3.71, "read": 3.71}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.138, "count": 110, "max_ms": 15.199, "p95_ms": 11.003}, "gstreamer_push": {"avg_ms": 0.923, "count": 111, "max_ms": 1.861, "p95_ms": 1.757}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.511, "count": 110, "max_ms": 73.493, "p95_ms": 70.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.407, "count": 110, "max_ms": 94.173, "p95_ms": 90.71}, "yolo11_640_fire_11class": {"avg_ms": 59.688, "count": 110, "max_ms": 77.854, "p95_ms": 73.637}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.66, "count": 110, "max_ms": 94.536, "p95_ms": 92.386}}, "truncated_count": 0}, "processing_total": {"avg_ms": 261.931, "count": 111, "max_ms": 340.248, "p95_ms": 284.212}, "read_interval": {"avg_ms": 271.578, "count": 111, "max_ms": 349.968, "p95_ms": 294.313}, "triton_wait": {"avg_ms": 76.007, "count": 110, "max_ms": 94.549, "p95_ms": 92.485}, "visualize": {"avg_ms": 11.997, "count": 110, "max_ms": 15.535, "p95_ms": 14.359}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 364}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3990.273, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:41:05+09:00", "last_error_summary": "[h264 @ 0x55f009a29b00] error while decoding MB 51 43, bytestream -32", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:41:05+09:00", "last_error_summary": "[h264 @ 0x55f009a29b00] error while decoding MB 51 43, bytestream -32", "last_exit_at": "2026-07-01T17:41:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.156, "last_started_at": "2026-07-01T17:39:14+09:00", "pid": 12905, "process": {"exists": false, "pid": 12905}, "restart_count": 86, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.948, "sample_count": 1120, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43717, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:41:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:41:05+09:00", "last_frame_read_age_sec": 0.272, "last_frame_read_at": "2026-07-01T17:41:04+09:00", "mjpeg": {"age_sec": 0.26, "exists": true, "mtime": "2026-07-01T17:41:04.999675+09:00", "mtime_epoch": 1782895264.9996748, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203261}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:41:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 888.09, "name": "celery", "pid": 237, "rss_mb": 881.977, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.718, "pushed": 3.718, "read": 3.685}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.132, "count": 110, "max_ms": 15.199, "p95_ms": 11.003}, "gstreamer_push": {"avg_ms": 0.923, "count": 111, "max_ms": 1.861, "p95_ms": 1.757}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.304, "count": 110, "max_ms": 73.493, "p95_ms": 70.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.401, "count": 110, "max_ms": 94.173, "p95_ms": 90.71}, "yolo11_640_fire_11class": {"avg_ms": 59.717, "count": 110, "max_ms": 77.854, "p95_ms": 73.637}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.534, "count": 110, "max_ms": 94.536, "p95_ms": 92.386}}, "truncated_count": 0}, "processing_total": {"avg_ms": 261.811, "count": 111, "max_ms": 340.248, "p95_ms": 284.212}, "read_interval": {"avg_ms": 271.399, "count": 110, "max_ms": 349.968, "p95_ms": 294.313}, "triton_wait": {"avg_ms": 75.992, "count": 110, "max_ms": 94.549, "p95_ms": 92.485}, "visualize": {"avg_ms": 11.997, "count": 110, "max_ms": 15.535, "p95_ms": 14.359}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.851, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 364}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3990.391, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:41:05+09:00", "last_error_summary": "[h264 @ 0x55f009a29b00] error while decoding MB 51 43, bytestream -32", "last_exit_at": "2026-07-01T17:41:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.159, "last_started_at": "2026-07-01T17:39:14+09:00", "pid": 12905, "process": {"exists": false, "pid": 12905}, "restart_count": 86, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.948, "sample_count": 1120, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43717, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:41:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:41:05+09:00", "last_frame_read_age_sec": 0.275, "last_frame_read_at": "2026-07-01T17:41:04+09:00", "mjpeg": {"age_sec": 0.264, "exists": true, "mtime": "2026-07-01T17:41:04.999675+09:00", "mtime_epoch": 1782895264.9996748, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203261}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:41:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 888.09, "name": "celery", "pid": 237, "rss_mb": 881.977, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.718, "pushed": 3.718, "read": 3.685}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.132, "count": 110, "max_ms": 15.199, "p95_ms": 11.003}, "gstreamer_push": {"avg_ms": 0.923, "count": 111, "max_ms": 1.861, "p95_ms": 1.757}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.304, "count": 110, "max_ms": 73.493, "p95_ms": 70.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.401, "count": 110, "max_ms": 94.173, "p95_ms": 90.71}, "yolo11_640_fire_11class": {"avg_ms": 59.717, "count": 110, "max_ms": 77.854, "p95_ms": 73.637}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.534, "count": 110, "max_ms": 94.536, "p95_ms": 92.386}}, "truncated_count": 0}, "processing_total": {"avg_ms": 261.811, "count": 111, "max_ms": 340.248, "p95_ms": 284.212}, "read_interval": {"avg_ms": 271.399, "count": 110, "max_ms": 349.968, "p95_ms": 294.313}, "triton_wait": {"avg_ms": 75.992, "count": 110, "max_ms": 94.549, "p95_ms": 92.485}, "visualize": {"avg_ms": 11.997, "count": 110, "max_ms": 15.535, "p95_ms": 14.359}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.854, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 364}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4104.379, "limit_mb": 8796093022207.996}, "detail": {"pid": 13012, "restart_count": 87, "start_count": 88}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:41:05+09:00", "last_error_summary": "[h264 @ 0x55f009a29b00] error while decoding MB 51 43, bytestream -32", "last_exit_at": "2026-07-01T17:41:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:41:07+09:00", "pid": 13012, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.199, "name": "ffmpeg", "pid": 13012, "rss_mb": 142.199, "state": "S (sleeping)", "threads": 106}, "restart_count": 87, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.891, "sample_count": 1103, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 43717, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:41:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.233, "last_frame_pushed_at": "2026-07-01T17:41:05+09:00", "last_frame_read_age_sec": 4.502, "last_frame_read_at": "2026-07-01T17:41:04+09:00", "mjpeg": {"age_sec": 4.214, "exists": true, "mtime": "2026-07-01T17:41:05.275679+09:00", "mtime_epoch": 1782895265.275679, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197960}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:41:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 888.09, "name": "celery", "pid": 237, "rss_mb": 881.977, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.179, "pushed": 3.179, "read": 3.146}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.121, "count": 94, "max_ms": 15.199, "p95_ms": 10.996}, "gstreamer_push": {"avg_ms": 0.923, "count": 95, "max_ms": 1.861, "p95_ms": 1.757}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.48, "count": 94, "max_ms": 73.493, "p95_ms": 70.284}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.841, "count": 94, "max_ms": 94.173, "p95_ms": 90.71}, "yolo11_640_fire_11class": {"avg_ms": 61.258, "count": 94, "max_ms": 77.854, "p95_ms": 74.115}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.34, "count": 94, "max_ms": 94.536, "p95_ms": 92.386}}, "truncated_count": 0}, "processing_total": {"avg_ms": 262.958, "count": 95, "max_ms": 340.248, "p95_ms": 284.212}, "read_interval": {"avg_ms": 272.564, "count": 94, "max_ms": 349.968, "p95_ms": 294.313}, "triton_wait": {"avg_ms": 77.787, "count": 94, "max_ms": 94.549, "p95_ms": 92.485}, "visualize": {"avg_ms": 12.021, "count": 94, "max_ms": 15.535, "p95_ms": 14.914}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 367}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3994.75, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55925cf3f680] error while decoding MB 11 29, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:42:58+09:00", "last_error_summary": "[h264 @ 0x55925cf3f680] error while decoding MB 11 29, bytestream -15", "last_exit_at": "2026-07-01T17:41:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.001, "last_started_at": "2026-07-01T17:41:07+09:00", "pid": 13012, "process": {"exists": false, "pid": 13012}, "restart_count": 87, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.148, "sample_count": 1119, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44136, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:42:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T17:42:58+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T17:42:58+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T17:42:58.445489+09:00", "mtime_epoch": 1782895378.4454892, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196715}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:42:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 888.09, "name": "celery", "pid": 237, "rss_mb": 882.613, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.803, "pushed": 3.803, "read": 3.803}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.294, "count": 113, "max_ms": 14.181, "p95_ms": 11.309}, "gstreamer_push": {"avg_ms": 0.919, "count": 114, "max_ms": 1.859, "p95_ms": 1.058}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.191, "count": 113, "max_ms": 71.805, "p95_ms": 66.321}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.706, "count": 113, "max_ms": 89.469, "p95_ms": 85.559}, "yolo11_640_fire_11class": {"avg_ms": 53.731, "count": 113, "max_ms": 74.211, "p95_ms": 72.078}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.946, "count": 113, "max_ms": 93.188, "p95_ms": 88.854}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.021, "count": 114, "max_ms": 344.995, "p95_ms": 288.12}, "read_interval": {"avg_ms": 264.654, "count": 114, "max_ms": 354.966, "p95_ms": 297.337}, "triton_wait": {"avg_ms": 68.236, "count": 113, "max_ms": 93.201, "p95_ms": 89.406}, "visualize": {"avg_ms": 11.826, "count": 113, "max_ms": 13.169, "p95_ms": 12.572}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 367}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3993.684, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:42:58+09:00", "last_error_summary": "[h264 @ 0x55925cf3f680] error while decoding MB 11 29, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:42:58+09:00", "last_error_summary": "[h264 @ 0x55925cf3f680] error while decoding MB 11 29, bytestream -15", "last_exit_at": "2026-07-01T17:42:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.216, "last_started_at": "2026-07-01T17:41:07+09:00", "pid": 13012, "process": {"exists": false, "pid": 13012}, "restart_count": 87, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.148, "sample_count": 1119, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44137, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:42:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:42:58+09:00", "last_frame_read_age_sec": 0.269, "last_frame_read_at": "2026-07-01T17:42:58+09:00", "mjpeg": {"age_sec": 0.26, "exists": true, "mtime": "2026-07-01T17:42:58.445489+09:00", "mtime_epoch": 1782895378.4454892, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196715}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:42:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 888.09, "name": "celery", "pid": 237, "rss_mb": 882.613, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.808, "pushed": 3.808, "read": 3.774}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.286, "count": 113, "max_ms": 14.181, "p95_ms": 11.309}, "gstreamer_push": {"avg_ms": 0.919, "count": 114, "max_ms": 1.859, "p95_ms": 1.058}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.385, "count": 113, "max_ms": 71.805, "p95_ms": 66.321}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.924, "count": 113, "max_ms": 89.469, "p95_ms": 85.559}, "yolo11_640_fire_11class": {"avg_ms": 53.944, "count": 113, "max_ms": 74.211, "p95_ms": 72.078}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.139, "count": 113, "max_ms": 93.188, "p95_ms": 88.854}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.188, "count": 114, "max_ms": 344.995, "p95_ms": 288.12}, "read_interval": {"avg_ms": 264.736, "count": 113, "max_ms": 354.966, "p95_ms": 297.337}, "triton_wait": {"avg_ms": 68.428, "count": 113, "max_ms": 93.201, "p95_ms": 89.406}, "visualize": {"avg_ms": 11.827, "count": 113, "max_ms": 13.169, "p95_ms": 12.572}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 367}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3993.676, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:42:58+09:00", "last_error_summary": "[h264 @ 0x55925cf3f680] error while decoding MB 11 29, bytestream -15", "last_exit_at": "2026-07-01T17:42:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.221, "last_started_at": "2026-07-01T17:41:07+09:00", "pid": 13012, "process": {"exists": false, "pid": 13012}, "restart_count": 87, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.148, "sample_count": 1119, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44137, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:42:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T17:42:58+09:00", "last_frame_read_age_sec": 0.273, "last_frame_read_at": "2026-07-01T17:42:58+09:00", "mjpeg": {"age_sec": 0.264, "exists": true, "mtime": "2026-07-01T17:42:58.445489+09:00", "mtime_epoch": 1782895378.4454892, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196715}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:42:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 888.09, "name": "celery", "pid": 237, "rss_mb": 882.613, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.807, "pushed": 3.807, "read": 3.774}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.286, "count": 113, "max_ms": 14.181, "p95_ms": 11.309}, "gstreamer_push": {"avg_ms": 0.919, "count": 114, "max_ms": 1.859, "p95_ms": 1.058}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.385, "count": 113, "max_ms": 71.805, "p95_ms": 66.321}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.924, "count": 113, "max_ms": 89.469, "p95_ms": 85.559}, "yolo11_640_fire_11class": {"avg_ms": 53.944, "count": 113, "max_ms": 74.211, "p95_ms": 72.078}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.139, "count": 113, "max_ms": 93.188, "p95_ms": 88.854}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.188, "count": 114, "max_ms": 344.995, "p95_ms": 288.12}, "read_interval": {"avg_ms": 264.736, "count": 113, "max_ms": 354.966, "p95_ms": 297.337}, "triton_wait": {"avg_ms": 68.428, "count": 113, "max_ms": 93.201, "p95_ms": 89.406}, "visualize": {"avg_ms": 11.827, "count": 113, "max_ms": 13.169, "p95_ms": 12.572}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 367}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4104.305, "limit_mb": 8796093022207.996}, "detail": {"pid": 13119, "restart_count": 88, "start_count": 89}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:42:58+09:00", "last_error_summary": "[h264 @ 0x55925cf3f680] error while decoding MB 11 29, bytestream -15", "last_exit_at": "2026-07-01T17:43:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:43:01+09:00", "pid": 13119, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.34, "name": "ffmpeg", "pid": 13119, "rss_mb": 139.34, "state": "S (sleeping)", "threads": 106}, "restart_count": 88, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.184, "sample_count": 1099, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44137, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:42:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.845, "last_frame_pushed_at": "2026-07-01T17:42:58+09:00", "last_frame_read_age_sec": 5.111, "last_frame_read_at": "2026-07-01T17:42:58+09:00", "mjpeg": {"age_sec": 4.83, "exists": true, "mtime": "2026-07-01T17:42:58.717494+09:00", "mtime_epoch": 1782895378.7174935, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191466}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:43:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 888.09, "name": "celery", "pid": 237, "rss_mb": 882.613, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.213, "pushed": 3.213, "read": 3.18}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.221, "count": 95, "max_ms": 14.181, "p95_ms": 11.231}, "gstreamer_push": {"avg_ms": 0.919, "count": 96, "max_ms": 1.856, "p95_ms": 1.058}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.575, "count": 95, "max_ms": 71.805, "p95_ms": 67.329}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.877, "count": 95, "max_ms": 89.469, "p95_ms": 85.559}, "yolo11_640_fire_11class": {"avg_ms": 53.127, "count": 95, "max_ms": 74.211, "p95_ms": 72.326}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.122, "count": 95, "max_ms": 91.389, "p95_ms": 88.854}}, "truncated_count": 0}, "processing_total": {"avg_ms": 253.678, "count": 96, "max_ms": 344.995, "p95_ms": 288.12}, "read_interval": {"avg_ms": 263.213, "count": 95, "max_ms": 354.966, "p95_ms": 297.337}, "triton_wait": {"avg_ms": 67.337, "count": 95, "max_ms": 91.402, "p95_ms": 89.315}, "visualize": {"avg_ms": 11.804, "count": 95, "max_ms": 12.781, "p95_ms": 12.432}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.875, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3993.16, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557475ec6080] error while decoding MB 54 21, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:44:53+09:00", "last_error_summary": "[h264 @ 0x557475ec6080] error while decoding MB 54 21, bytestream -5", "last_exit_at": "2026-07-01T17:43:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.58, "last_started_at": "2026-07-01T17:43:01+09:00", "pid": 13119, "process": {"exists": false, "pid": 13119}, "restart_count": 88, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.891, "sample_count": 1108, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44554, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:44:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T17:44:53+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-01T17:44:53+09:00", "mjpeg": {"age_sec": 0.035, "exists": true, "mtime": "2026-07-01T17:44:53.091337+09:00", "mtime_epoch": 1782895493.091337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190962}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:44:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.141, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.812, "pushed": 3.812, "read": 3.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.225, "count": 113, "max_ms": 16.285, "p95_ms": 10.599}, "gstreamer_push": {"avg_ms": 0.94, "count": 114, "max_ms": 3.069, "p95_ms": 1.758}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.396, "count": 113, "max_ms": 71.051, "p95_ms": 64.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.866, "count": 113, "max_ms": 90.255, "p95_ms": 87.166}, "yolo11_640_fire_11class": {"avg_ms": 55.29, "count": 113, "max_ms": 76.005, "p95_ms": 71.815}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.308, "count": 113, "max_ms": 90.859, "p95_ms": 90.104}}, "truncated_count": 0}, "processing_total": {"avg_ms": 254.352, "count": 114, "max_ms": 345.086, "p95_ms": 277.222}, "read_interval": {"avg_ms": 264.065, "count": 114, "max_ms": 357.846, "p95_ms": 287.591}, "triton_wait": {"avg_ms": 69.854, "count": 113, "max_ms": 90.874, "p95_ms": 90.14}, "visualize": {"avg_ms": 11.874, "count": 113, "max_ms": 13.923, "p95_ms": 12.689}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3994.449, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:44:53+09:00", "last_error_summary": "[h264 @ 0x557475ec6080] error while decoding MB 54 21, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:44:53+09:00", "last_error_summary": "[h264 @ 0x557475ec6080] error while decoding MB 54 21, bytestream -5", "last_exit_at": "2026-07-01T17:44:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.774, "last_started_at": "2026-07-01T17:43:01+09:00", "pid": 13119, "process": {"exists": false, "pid": 13119}, "restart_count": 88, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.891, "sample_count": 1108, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44555, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:44:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:44:53+09:00", "last_frame_read_age_sec": 0.243, "last_frame_read_at": "2026-07-01T17:44:53+09:00", "mjpeg": {"age_sec": 0.23, "exists": true, "mtime": "2026-07-01T17:44:53.091337+09:00", "mtime_epoch": 1782895493.091337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190962}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:44:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.141, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.82, "pushed": 3.82, "read": 3.786}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.222, "count": 114, "max_ms": 16.285, "p95_ms": 10.599}, "gstreamer_push": {"avg_ms": 0.94, "count": 114, "max_ms": 3.069, "p95_ms": 1.758}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.317, "count": 114, "max_ms": 71.051, "p95_ms": 64.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.754, "count": 114, "max_ms": 90.255, "p95_ms": 87.166}, "yolo11_640_fire_11class": {"avg_ms": 55.201, "count": 114, "max_ms": 76.005, "p95_ms": 71.815}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.202, "count": 114, "max_ms": 90.859, "p95_ms": 90.104}}, "truncated_count": 0}, "processing_total": {"avg_ms": 254.297, "count": 114, "max_ms": 345.086, "p95_ms": 277.222}, "read_interval": {"avg_ms": 264.137, "count": 113, "max_ms": 357.846, "p95_ms": 287.591}, "triton_wait": {"avg_ms": 69.744, "count": 114, "max_ms": 90.874, "p95_ms": 90.14}, "visualize": {"avg_ms": 11.875, "count": 114, "max_ms": 13.923, "p95_ms": 12.689}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3994.566, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:44:53+09:00", "last_error_summary": "[h264 @ 0x557475ec6080] error while decoding MB 54 21, bytestream -5", "last_exit_at": "2026-07-01T17:44:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.778, "last_started_at": "2026-07-01T17:43:01+09:00", "pid": 13119, "process": {"exists": false, "pid": 13119}, "restart_count": 88, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.891, "sample_count": 1108, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44555, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:44:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:44:53+09:00", "last_frame_read_age_sec": 0.246, "last_frame_read_at": "2026-07-01T17:44:53+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-01T17:44:53.091337+09:00", "mtime_epoch": 1782895493.091337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190962}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:44:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.141, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.819, "pushed": 3.819, "read": 3.786}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.226, "count": 113, "max_ms": 16.285, "p95_ms": 10.599}, "gstreamer_push": {"avg_ms": 0.94, "count": 114, "max_ms": 3.069, "p95_ms": 1.758}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.314, "count": 113, "max_ms": 71.051, "p95_ms": 64.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.751, "count": 113, "max_ms": 90.255, "p95_ms": 87.166}, "yolo11_640_fire_11class": {"avg_ms": 55.2, "count": 113, "max_ms": 76.005, "p95_ms": 71.815}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.204, "count": 113, "max_ms": 90.859, "p95_ms": 90.104}}, "truncated_count": 0}, "processing_total": {"avg_ms": 254.297, "count": 114, "max_ms": 345.086, "p95_ms": 277.222}, "read_interval": {"avg_ms": 264.137, "count": 113, "max_ms": 357.846, "p95_ms": 287.591}, "triton_wait": {"avg_ms": 69.751, "count": 113, "max_ms": 90.874, "p95_ms": 90.14}, "visualize": {"avg_ms": 11.88, "count": 113, "max_ms": 13.923, "p95_ms": 12.689}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.848, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 371}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4103.273, "limit_mb": 8796093022207.996}, "detail": {"pid": 13226, "restart_count": 89, "start_count": 90}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:44:53+09:00", "last_error_summary": "[h264 @ 0x557475ec6080] error while decoding MB 54 21, bytestream -5", "last_exit_at": "2026-07-01T17:44:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:44:55+09:00", "pid": 13226, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.035, "name": "ffmpeg", "pid": 13226, "rss_mb": 139.035, "state": "S (sleeping)", "threads": 106}, "restart_count": 89, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.619, "sample_count": 1092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44555, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:44:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.128, "last_frame_pushed_at": "2026-07-01T17:44:53+09:00", "last_frame_read_age_sec": 4.368, "last_frame_read_at": "2026-07-01T17:44:53+09:00", "mjpeg": {"age_sec": 4.111, "exists": true, "mtime": "2026-07-01T17:44:53.335341+09:00", "mtime_epoch": 1782895493.335341, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191142}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:44:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.141, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.3, "pushed": 3.3, "read": 3.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.249, "count": 98, "max_ms": 16.285, "p95_ms": 11.057}, "gstreamer_push": {"avg_ms": 0.94, "count": 99, "max_ms": 3.069, "p95_ms": 1.758}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.043, "count": 98, "max_ms": 71.051, "p95_ms": 64.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.396, "count": 98, "max_ms": 90.255, "p95_ms": 87.599}, "yolo11_640_fire_11class": {"avg_ms": 54.916, "count": 98, "max_ms": 76.005, "p95_ms": 71.815}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.843, "count": 98, "max_ms": 90.859, "p95_ms": 90.128}}, "truncated_count": 0}, "processing_total": {"avg_ms": 254.155, "count": 99, "max_ms": 345.086, "p95_ms": 278.316}, "read_interval": {"avg_ms": 263.976, "count": 98, "max_ms": 357.846, "p95_ms": 288.105}, "triton_wait": {"avg_ms": 69.438, "count": 98, "max_ms": 90.874, "p95_ms": 90.268}, "visualize": {"avg_ms": 11.885, "count": 98, "max_ms": 13.923, "p95_ms": 12.988}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 374}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3994.227, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d485c866c0] error while decoding MB 31 35, bytestream -38", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:46:50+09:00", "last_error_summary": "[h264 @ 0x55d485c866c0] error while decoding MB 31 35, bytestream -38", "last_exit_at": "2026-07-01T17:44:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.42, "last_started_at": "2026-07-01T17:44:55+09:00", "pid": 13226, "process": {"exists": false, "pid": 13226}, "restart_count": 89, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.536, "sample_count": 1117, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44994, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:46:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T17:46:50+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T17:46:50+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T17:46:50.820831+09:00", "mtime_epoch": 1782895610.8208313, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:46:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.406, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.834, "pushed": 3.834, "read": 3.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.048, "count": 114, "max_ms": 17.378, "p95_ms": 10.982}, "gstreamer_push": {"avg_ms": 0.938, "count": 115, "max_ms": 2.795, "p95_ms": 1.826}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.143, "count": 114, "max_ms": 72.4, "p95_ms": 64.471}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.269, "count": 114, "max_ms": 89.63, "p95_ms": 86.487}, "yolo11_640_fire_11class": {"avg_ms": 55.853, "count": 114, "max_ms": 81.576, "p95_ms": 71.498}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.322, "count": 114, "max_ms": 97.165, "p95_ms": 90.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 253.113, "count": 115, "max_ms": 352.054, "p95_ms": 295.445}, "read_interval": {"avg_ms": 262.746, "count": 115, "max_ms": 362.153, "p95_ms": 305.569}, "triton_wait": {"avg_ms": 70.813, "count": 114, "max_ms": 97.177, "p95_ms": 90.391}, "visualize": {"avg_ms": 11.707, "count": 114, "max_ms": 15.42, "p95_ms": 12.79}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 374}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3993.094, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:46:50+09:00", "last_error_summary": "[h264 @ 0x55d485c866c0] error while decoding MB 31 35, bytestream -38", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:46:50+09:00", "last_error_summary": "[h264 @ 0x55d485c866c0] error while decoding MB 31 35, bytestream -38", "last_exit_at": "2026-07-01T17:46:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.628, "last_started_at": "2026-07-01T17:44:55+09:00", "pid": 13226, "process": {"exists": false, "pid": 13226}, "restart_count": 89, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.536, "sample_count": 1117, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44995, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:46:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:46:51+09:00", "last_frame_read_age_sec": 0.259, "last_frame_read_at": "2026-07-01T17:46:50+09:00", "mjpeg": {"age_sec": 0.253, "exists": true, "mtime": "2026-07-01T17:46:50.820831+09:00", "mtime_epoch": 1782895610.8208313, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:46:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.406, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.843, "pushed": 3.843, "read": 3.81}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.059, "count": 114, "max_ms": 17.378, "p95_ms": 10.982}, "gstreamer_push": {"avg_ms": 0.938, "count": 115, "max_ms": 2.795, "p95_ms": 1.826}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.918, "count": 114, "max_ms": 72.4, "p95_ms": 64.471}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.149, "count": 114, "max_ms": 89.63, "p95_ms": 86.487}, "yolo11_640_fire_11class": {"avg_ms": 55.684, "count": 114, "max_ms": 81.576, "p95_ms": 71.334}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.2, "count": 114, "max_ms": 97.165, "p95_ms": 90.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 252.959, "count": 115, "max_ms": 352.054, "p95_ms": 295.445}, "read_interval": {"avg_ms": 262.577, "count": 114, "max_ms": 362.153, "p95_ms": 305.569}, "triton_wait": {"avg_ms": 70.691, "count": 114, "max_ms": 97.177, "p95_ms": 90.391}, "visualize": {"avg_ms": 11.707, "count": 114, "max_ms": 15.42, "p95_ms": 12.79}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 374}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3993.086, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:46:50+09:00", "last_error_summary": "[h264 @ 0x55d485c866c0] error while decoding MB 31 35, bytestream -38", "last_exit_at": "2026-07-01T17:46:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.631, "last_started_at": "2026-07-01T17:44:55+09:00", "pid": 13226, "process": {"exists": false, "pid": 13226}, "restart_count": 89, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.536, "sample_count": 1117, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44995, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:46:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:46:51+09:00", "last_frame_read_age_sec": 0.263, "last_frame_read_at": "2026-07-01T17:46:50+09:00", "mjpeg": {"age_sec": 0.256, "exists": true, "mtime": "2026-07-01T17:46:50.820831+09:00", "mtime_epoch": 1782895610.8208313, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:46:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.406, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.843, "pushed": 3.843, "read": 3.809}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.059, "count": 114, "max_ms": 17.378, "p95_ms": 10.982}, "gstreamer_push": {"avg_ms": 0.938, "count": 115, "max_ms": 2.795, "p95_ms": 1.826}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.918, "count": 114, "max_ms": 72.4, "p95_ms": 64.471}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.149, "count": 114, "max_ms": 89.63, "p95_ms": 86.487}, "yolo11_640_fire_11class": {"avg_ms": 55.684, "count": 114, "max_ms": 81.576, "p95_ms": 71.334}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.2, "count": 114, "max_ms": 97.165, "p95_ms": 90.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 252.959, "count": 115, "max_ms": 352.054, "p95_ms": 295.445}, "read_interval": {"avg_ms": 262.577, "count": 114, "max_ms": 362.153, "p95_ms": 305.569}, "triton_wait": {"avg_ms": 70.691, "count": 114, "max_ms": 97.177, "p95_ms": 90.391}, "visualize": {"avg_ms": 11.707, "count": 114, "max_ms": 15.42, "p95_ms": 12.79}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 374}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4098.523, "limit_mb": 8796093022207.996}, "detail": {"pid": 13333, "restart_count": 90, "start_count": 91}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:46:50+09:00", "last_error_summary": "[h264 @ 0x55d485c866c0] error while decoding MB 31 35, bytestream -38", "last_exit_at": "2026-07-01T17:46:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:46:53+09:00", "pid": 13333, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.648, "name": "ffmpeg", "pid": 13333, "rss_mb": 136.648, "state": "S (sleeping)", "threads": 106}, "restart_count": 90, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.536, "sample_count": 1101, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 44995, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:46:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.156, "last_frame_pushed_at": "2026-07-01T17:46:51+09:00", "last_frame_read_age_sec": 4.412, "last_frame_read_at": "2026-07-01T17:46:50+09:00", "mjpeg": {"age_sec": 4.142, "exists": true, "mtime": "2026-07-01T17:46:51.084834+09:00", "mtime_epoch": 1782895611.0848339, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188163}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:46:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.406, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.322, "pushed": 3.322, "read": 3.288}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.076, "count": 99, "max_ms": 11.502, "p95_ms": 10.982}, "gstreamer_push": {"avg_ms": 0.931, "count": 99, "max_ms": 2.795, "p95_ms": 1.798}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.12, "count": 99, "max_ms": 72.4, "p95_ms": 62.436}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.911, "count": 99, "max_ms": 89.63, "p95_ms": 87.898}, "yolo11_640_fire_11class": {"avg_ms": 55.249, "count": 99, "max_ms": 81.576, "p95_ms": 70.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.731, "count": 99, "max_ms": 97.165, "p95_ms": 89.162}}, "truncated_count": 0}, "processing_total": {"avg_ms": 251.813, "count": 99, "max_ms": 352.054, "p95_ms": 295.445}, "read_interval": {"avg_ms": 261.411, "count": 98, "max_ms": 362.153, "p95_ms": 305.569}, "triton_wait": {"avg_ms": 70.295, "count": 99, "max_ms": 97.177, "p95_ms": 89.175}, "visualize": {"avg_ms": 11.669, "count": 99, "max_ms": 13.485, "p95_ms": 12.79}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.802, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 378}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3994.91, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55940be5e940] error while decoding MB 79 34, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:48:46+09:00", "last_error_summary": "[h264 @ 0x55940be5e940] error while decoding MB 79 34, bytestream -7", "last_exit_at": "2026-07-01T17:46:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.996, "last_started_at": "2026-07-01T17:46:53+09:00", "pid": 13333, "process": {"exists": false, "pid": 13333}, "restart_count": 90, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 280.283, "sample_count": 1127, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45425, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:48:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T17:48:46+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T17:48:46+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T17:48:46.174004+09:00", "mtime_epoch": 1782895726.1740036, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184092}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:48:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.16, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.95, "pushed": 3.95, "read": 3.95}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.153, "count": 117, "max_ms": 17.953, "p95_ms": 14.846}, "gstreamer_push": {"avg_ms": 0.947, "count": 118, "max_ms": 3.855, "p95_ms": 1.358}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.58, "count": 117, "max_ms": 70.031, "p95_ms": 59.043}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.879, "count": 117, "max_ms": 89.894, "p95_ms": 81.943}, "yolo11_640_fire_11class": {"avg_ms": 51.222, "count": 117, "max_ms": 71.122, "p95_ms": 69.493}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.952, "count": 117, "max_ms": 90.627, "p95_ms": 86.994}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.965, "count": 118, "max_ms": 336.246, "p95_ms": 292.239}, "read_interval": {"avg_ms": 254.685, "count": 118, "max_ms": 347.642, "p95_ms": 301.199}, "triton_wait": {"avg_ms": 65.202, "count": 117, "max_ms": 90.641, "p95_ms": 87.006}, "visualize": {"avg_ms": 11.783, "count": 117, "max_ms": 18.348, "p95_ms": 15.38}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 378}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3993.379, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:48:46+09:00", "last_error_summary": "[h264 @ 0x55940be5e940] error while decoding MB 79 34, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:48:46+09:00", "last_error_summary": "[h264 @ 0x55940be5e940] error while decoding MB 79 34, bytestream -7", "last_exit_at": "2026-07-01T17:48:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.242, "last_started_at": "2026-07-01T17:46:53+09:00", "pid": 13333, "process": {"exists": false, "pid": 13333}, "restart_count": 90, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 280.558, "sample_count": 1127, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45426, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:48:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T17:48:46+09:00", "last_frame_read_age_sec": 0.3, "last_frame_read_at": "2026-07-01T17:48:46+09:00", "mjpeg": {"age_sec": 0.294, "exists": true, "mtime": "2026-07-01T17:48:46.174004+09:00", "mtime_epoch": 1782895726.1740036, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184092}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:48:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.16, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.95, "pushed": 3.95, "read": 3.916}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.197, "count": 117, "max_ms": 17.953, "p95_ms": 14.846}, "gstreamer_push": {"avg_ms": 0.947, "count": 118, "max_ms": 3.855, "p95_ms": 1.358}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.624, "count": 117, "max_ms": 70.031, "p95_ms": 59.043}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.897, "count": 117, "max_ms": 89.894, "p95_ms": 81.943}, "yolo11_640_fire_11class": {"avg_ms": 51.24, "count": 117, "max_ms": 71.122, "p95_ms": 69.493}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.97, "count": 117, "max_ms": 90.627, "p95_ms": 86.994}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.475, "count": 118, "max_ms": 336.246, "p95_ms": 293.921}, "read_interval": {"avg_ms": 254.787, "count": 117, "max_ms": 347.642, "p95_ms": 301.199}, "triton_wait": {"avg_ms": 65.22, "count": 117, "max_ms": 90.641, "p95_ms": 87.006}, "visualize": {"avg_ms": 11.812, "count": 117, "max_ms": 18.348, "p95_ms": 15.38}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 378}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3993.371, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:48:46+09:00", "last_error_summary": "[h264 @ 0x55940be5e940] error while decoding MB 79 34, bytestream -7", "last_exit_at": "2026-07-01T17:48:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.245, "last_started_at": "2026-07-01T17:46:53+09:00", "pid": 13333, "process": {"exists": false, "pid": 13333}, "restart_count": 90, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 91}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 280.558, "sample_count": 1127, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45426, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:48:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T17:48:46+09:00", "last_frame_read_age_sec": 0.303, "last_frame_read_at": "2026-07-01T17:48:46+09:00", "mjpeg": {"age_sec": 0.297, "exists": true, "mtime": "2026-07-01T17:48:46.174004+09:00", "mtime_epoch": 1782895726.1740036, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184092}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:48:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.16, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.949, "pushed": 3.949, "read": 3.916}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.197, "count": 117, "max_ms": 17.953, "p95_ms": 14.846}, "gstreamer_push": {"avg_ms": 0.947, "count": 118, "max_ms": 3.855, "p95_ms": 1.358}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.624, "count": 117, "max_ms": 70.031, "p95_ms": 59.043}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.897, "count": 117, "max_ms": 89.894, "p95_ms": 81.943}, "yolo11_640_fire_11class": {"avg_ms": 51.24, "count": 117, "max_ms": 71.122, "p95_ms": 69.493}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.97, "count": 117, "max_ms": 90.627, "p95_ms": 86.994}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.475, "count": 118, "max_ms": 336.246, "p95_ms": 293.921}, "read_interval": {"avg_ms": 254.787, "count": 117, "max_ms": 347.642, "p95_ms": 301.199}, "triton_wait": {"avg_ms": 65.22, "count": 117, "max_ms": 90.641, "p95_ms": 87.006}, "visualize": {"avg_ms": 11.812, "count": 117, "max_ms": 18.348, "p95_ms": 15.38}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 378}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4104.328, "limit_mb": 8796093022207.996}, "detail": {"pid": 13440, "restart_count": 91, "start_count": 92}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:48:46+09:00", "last_error_summary": "[h264 @ 0x55940be5e940] error while decoding MB 79 34, bytestream -7", "last_exit_at": "2026-07-01T17:48:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:48:48+09:00", "pid": 13440, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.0, "name": "ffmpeg", "pid": 13440, "rss_mb": 142.0, "state": "S (sleeping)", "threads": 106}, "restart_count": 91, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 280.558, "sample_count": 1110, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45426, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:48:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.137, "last_frame_pushed_at": "2026-07-01T17:48:46+09:00", "last_frame_read_age_sec": 4.432, "last_frame_read_at": "2026-07-01T17:48:46+09:00", "mjpeg": {"age_sec": 4.119, "exists": true, "mtime": "2026-07-01T17:48:46.482007+09:00", "mtime_epoch": 1782895726.4820068, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:48:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.16, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.389, "pushed": 3.389, "read": 3.355}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.198, "count": 101, "max_ms": 17.953, "p95_ms": 14.622}, "gstreamer_push": {"avg_ms": 0.958, "count": 101, "max_ms": 3.855, "p95_ms": 1.358}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.315, "count": 101, "max_ms": 70.031, "p95_ms": 59.043}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.476, "count": 101, "max_ms": 89.894, "p95_ms": 81.943}, "yolo11_640_fire_11class": {"avg_ms": 51.782, "count": 101, "max_ms": 71.122, "p95_ms": 69.493}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.552, "count": 101, "max_ms": 90.627, "p95_ms": 86.994}}, "truncated_count": 0}, "processing_total": {"avg_ms": 246.661, "count": 101, "max_ms": 336.246, "p95_ms": 292.239}, "read_interval": {"avg_ms": 255.96, "count": 100, "max_ms": 347.642, "p95_ms": 300.791}, "triton_wait": {"avg_ms": 65.827, "count": 101, "max_ms": 90.641, "p95_ms": 87.006}, "visualize": {"avg_ms": 11.884, "count": 101, "max_ms": 18.348, "p95_ms": 15.38}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.805, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 381}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3997.629, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564f4fa64b80] error while decoding MB 50 7, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:50:28+09:00", "last_error_summary": "[h264 @ 0x564f4fa64b80] error while decoding MB 50 7, bytestream -7", "last_exit_at": "2026-07-01T17:48:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 99.685, "last_started_at": "2026-07-01T17:48:48+09:00", "pid": 13440, "process": {"exists": false, "pid": 13440}, "restart_count": 91, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 279.687, "sample_count": 1134, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45807, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:50:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T17:50:28+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T17:50:28+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T17:50:28.243143+09:00", "mtime_epoch": 1782895828.243143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:50:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.781, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.007, "pushed": 4.007, "read": 4.007}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.797, "count": 119, "max_ms": 12.49, "p95_ms": 10.676}, "gstreamer_push": {"avg_ms": 0.915, "count": 120, "max_ms": 2.912, "p95_ms": 1.208}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.458, "count": 119, "max_ms": 70.373, "p95_ms": 63.546}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.992, "count": 119, "max_ms": 92.755, "p95_ms": 86.885}, "yolo11_640_fire_11class": {"avg_ms": 52.529, "count": 119, "max_ms": 75.109, "p95_ms": 68.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.924, "count": 119, "max_ms": 93.785, "p95_ms": 87.958}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.656, "count": 120, "max_ms": 295.839, "p95_ms": 275.125}, "read_interval": {"avg_ms": 251.18, "count": 120, "max_ms": 305.982, "p95_ms": 284.175}, "triton_wait": {"avg_ms": 67.37, "count": 119, "max_ms": 93.797, "p95_ms": 88.555}, "visualize": {"avg_ms": 11.323, "count": 119, "max_ms": 15.714, "p95_ms": 12.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 381}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3996.707, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:50:28+09:00", "last_error_summary": "[h264 @ 0x564f4fa64b80] error while decoding MB 50 7, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:50:28+09:00", "last_error_summary": "[h264 @ 0x564f4fa64b80] error while decoding MB 50 7, bytestream -7", "last_exit_at": "2026-07-01T17:50:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 99.86, "last_started_at": "2026-07-01T17:48:48+09:00", "pid": 13440, "process": {"exists": false, "pid": 13440}, "restart_count": 91, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 279.687, "sample_count": 1135, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45808, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:50:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:50:28+09:00", "last_frame_read_age_sec": 0.226, "last_frame_read_at": "2026-07-01T17:50:28+09:00", "mjpeg": {"age_sec": 0.217, "exists": true, "mtime": "2026-07-01T17:50:28.243143+09:00", "mtime_epoch": 1782895828.243143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:50:28+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.781, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.018, "pushed": 4.018, "read": 3.984}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.791, "count": 119, "max_ms": 12.49, "p95_ms": 10.676}, "gstreamer_push": {"avg_ms": 0.915, "count": 120, "max_ms": 2.912, "p95_ms": 1.208}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.551, "count": 119, "max_ms": 70.373, "p95_ms": 63.546}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.998, "count": 119, "max_ms": 92.755, "p95_ms": 86.885}, "yolo11_640_fire_11class": {"avg_ms": 52.543, "count": 119, "max_ms": 75.109, "p95_ms": 68.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.94, "count": 119, "max_ms": 93.785, "p95_ms": 87.958}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.45, "count": 120, "max_ms": 295.839, "p95_ms": 275.125}, "read_interval": {"avg_ms": 251.137, "count": 119, "max_ms": 305.982, "p95_ms": 285.341}, "triton_wait": {"avg_ms": 67.376, "count": 119, "max_ms": 93.797, "p95_ms": 88.555}, "visualize": {"avg_ms": 11.314, "count": 119, "max_ms": 15.714, "p95_ms": 12.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 381}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3996.699, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:50:28+09:00", "last_error_summary": "[h264 @ 0x564f4fa64b80] error while decoding MB 50 7, bytestream -7", "last_exit_at": "2026-07-01T17:50:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 99.863, "last_started_at": "2026-07-01T17:48:48+09:00", "pid": 13440, "process": {"exists": false, "pid": 13440}, "restart_count": 91, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 92}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 279.687, "sample_count": 1135, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45808, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:50:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:50:28+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T17:50:28+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-01T17:50:28.243143+09:00", "mtime_epoch": 1782895828.243143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:50:28+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.781, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.017, "pushed": 4.017, "read": 3.984}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.791, "count": 119, "max_ms": 12.49, "p95_ms": 10.676}, "gstreamer_push": {"avg_ms": 0.915, "count": 120, "max_ms": 2.912, "p95_ms": 1.208}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.551, "count": 119, "max_ms": 70.373, "p95_ms": 63.546}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.998, "count": 119, "max_ms": 92.755, "p95_ms": 86.885}, "yolo11_640_fire_11class": {"avg_ms": 52.543, "count": 119, "max_ms": 75.109, "p95_ms": 68.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.94, "count": 119, "max_ms": 93.785, "p95_ms": 87.958}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.45, "count": 120, "max_ms": 295.839, "p95_ms": 275.125}, "read_interval": {"avg_ms": 251.137, "count": 119, "max_ms": 305.982, "p95_ms": 285.341}, "triton_wait": {"avg_ms": 67.376, "count": 119, "max_ms": 93.797, "p95_ms": 88.555}, "visualize": {"avg_ms": 11.314, "count": 119, "max_ms": 15.714, "p95_ms": 12.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 381}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4104.508, "limit_mb": 8796093022207.996}, "detail": {"pid": 13547, "restart_count": 92, "start_count": 93}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:50:28+09:00", "last_error_summary": "[h264 @ 0x564f4fa64b80] error while decoding MB 50 7, bytestream -7", "last_exit_at": "2026-07-01T17:50:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:50:31+09:00", "pid": 13547, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.398, "name": "ffmpeg", "pid": 13547, "rss_mb": 138.398, "state": "S (sleeping)", "threads": 106}, "restart_count": 92, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 280.165, "sample_count": 1116, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 45808, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:50:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.577, "last_frame_pushed_at": "2026-07-01T17:50:28+09:00", "last_frame_read_age_sec": 4.8, "last_frame_read_at": "2026-07-01T17:50:28+09:00", "mjpeg": {"age_sec": 4.559, "exists": true, "mtime": "2026-07-01T17:50:28.475146+09:00", "mtime_epoch": 1782895828.4751458, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172638}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:50:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.781, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.408, "pushed": 3.408, "read": 3.375}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.756, "count": 101, "max_ms": 12.49, "p95_ms": 10.676}, "gstreamer_push": {"avg_ms": 0.916, "count": 102, "max_ms": 2.912, "p95_ms": 1.208}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.493, "count": 101, "max_ms": 70.373, "p95_ms": 63.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.428, "count": 101, "max_ms": 90.278, "p95_ms": 85.1}, "yolo11_640_fire_11class": {"avg_ms": 52.089, "count": 101, "max_ms": 70.755, "p95_ms": 68.082}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.266, "count": 101, "max_ms": 91.373, "p95_ms": 87.704}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.467, "count": 102, "max_ms": 295.839, "p95_ms": 275.125}, "read_interval": {"avg_ms": 251.193, "count": 101, "max_ms": 305.982, "p95_ms": 284.175}, "triton_wait": {"avg_ms": 66.743, "count": 101, "max_ms": 91.386, "p95_ms": 87.717}, "visualize": {"avg_ms": 11.357, "count": 101, "max_ms": 15.714, "p95_ms": 12.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 385}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3998.91, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557be5ecb580] error while decoding MB 67 13, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:52:21+09:00", "last_error_summary": "[h264 @ 0x557be5ecb580] error while decoding MB 67 13, bytestream -5", "last_exit_at": "2026-07-01T17:50:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.034, "last_started_at": "2026-07-01T17:50:31+09:00", "pid": 13547, "process": {"exists": false, "pid": 13547}, "restart_count": 92, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.946, "sample_count": 1147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46242, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:52:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T17:52:21+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T17:52:21+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T17:52:21.020492+09:00", "mtime_epoch": 1782895941.0204923, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156884}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:52:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.014, "pushed": 4.014, "read": 4.014}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.781, "count": 119, "max_ms": 13.201, "p95_ms": 10.79}, "gstreamer_push": {"avg_ms": 0.96, "count": 120, "max_ms": 3.12, "p95_ms": 1.808}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.134, "count": 119, "max_ms": 68.993, "p95_ms": 62.986}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.329, "count": 119, "max_ms": 93.391, "p95_ms": 85.549}, "yolo11_640_fire_11class": {"avg_ms": 53.747, "count": 119, "max_ms": 82.263, "p95_ms": 71.518}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.44, "count": 119, "max_ms": 95.742, "p95_ms": 88.811}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.936, "count": 120, "max_ms": 321.911, "p95_ms": 270.797}, "read_interval": {"avg_ms": 250.647, "count": 120, "max_ms": 332.254, "p95_ms": 279.885}, "triton_wait": {"avg_ms": 67.979, "count": 119, "max_ms": 95.755, "p95_ms": 88.823}, "visualize": {"avg_ms": 11.164, "count": 119, "max_ms": 13.517, "p95_ms": 12.143}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 385}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3998.102, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:52:21+09:00", "last_error_summary": "[h264 @ 0x557be5ecb580] error while decoding MB 67 13, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:52:21+09:00", "last_error_summary": "[h264 @ 0x557be5ecb580] error while decoding MB 67 13, bytestream -5", "last_exit_at": "2026-07-01T17:52:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.246, "last_started_at": "2026-07-01T17:50:31+09:00", "pid": 13547, "process": {"exists": false, "pid": 13547}, "restart_count": 92, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.946, "sample_count": 1147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:52:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T17:52:21+09:00", "last_frame_read_age_sec": 0.265, "last_frame_read_at": "2026-07-01T17:52:21+09:00", "mjpeg": {"age_sec": 0.258, "exists": true, "mtime": "2026-07-01T17:52:21.020492+09:00", "mtime_epoch": 1782895941.0204923, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156884}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:52:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.25, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.019, "pushed": 4.019, "read": 3.986}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.787, "count": 119, "max_ms": 13.201, "p95_ms": 10.79}, "gstreamer_push": {"avg_ms": 0.96, "count": 120, "max_ms": 3.12, "p95_ms": 1.808}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.185, "count": 119, "max_ms": 68.993, "p95_ms": 62.986}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.377, "count": 119, "max_ms": 93.391, "p95_ms": 85.549}, "yolo11_640_fire_11class": {"avg_ms": 53.786, "count": 119, "max_ms": 82.263, "p95_ms": 71.518}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.501, "count": 119, "max_ms": 95.742, "p95_ms": 88.811}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.153, "count": 120, "max_ms": 321.911, "p95_ms": 270.797}, "read_interval": {"avg_ms": 250.712, "count": 119, "max_ms": 332.254, "p95_ms": 280.653}, "triton_wait": {"avg_ms": 68.033, "count": 119, "max_ms": 95.755, "p95_ms": 88.823}, "visualize": {"avg_ms": 11.171, "count": 119, "max_ms": 13.517, "p95_ms": 12.143}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.858, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 385}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3998.023, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:52:21+09:00", "last_error_summary": "[h264 @ 0x557be5ecb580] error while decoding MB 67 13, bytestream -5", "last_exit_at": "2026-07-01T17:52:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.249, "last_started_at": "2026-07-01T17:50:31+09:00", "pid": 13547, "process": {"exists": false, "pid": 13547}, "restart_count": 92, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 93}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.946, "sample_count": 1147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:52:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T17:52:21+09:00", "last_frame_read_age_sec": 0.268, "last_frame_read_at": "2026-07-01T17:52:21+09:00", "mjpeg": {"age_sec": 0.261, "exists": true, "mtime": "2026-07-01T17:52:21.020492+09:00", "mtime_epoch": 1782895941.0204923, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156884}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:52:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.25, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.019, "pushed": 4.019, "read": 3.985}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.787, "count": 119, "max_ms": 13.201, "p95_ms": 10.79}, "gstreamer_push": {"avg_ms": 0.96, "count": 120, "max_ms": 3.12, "p95_ms": 1.808}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.185, "count": 119, "max_ms": 68.993, "p95_ms": 62.986}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.377, "count": 119, "max_ms": 93.391, "p95_ms": 85.549}, "yolo11_640_fire_11class": {"avg_ms": 53.786, "count": 119, "max_ms": 82.263, "p95_ms": 71.518}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.501, "count": 119, "max_ms": 95.742, "p95_ms": 88.811}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.153, "count": 120, "max_ms": 321.911, "p95_ms": 270.797}, "read_interval": {"avg_ms": 250.712, "count": 119, "max_ms": 332.254, "p95_ms": 280.653}, "triton_wait": {"avg_ms": 68.033, "count": 119, "max_ms": 95.755, "p95_ms": 88.823}, "visualize": {"avg_ms": 11.171, "count": 119, "max_ms": 13.517, "p95_ms": 12.143}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.861, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 385}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4101.348, "limit_mb": 8796093022207.996}, "detail": {"pid": 13654, "restart_count": 93, "start_count": 94}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:52:21+09:00", "last_error_summary": "[h264 @ 0x557be5ecb580] error while decoding MB 67 13, bytestream -5", "last_exit_at": "2026-07-01T17:52:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:52:24+09:00", "pid": 13654, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.383, "name": "ffmpeg", "pid": 13654, "rss_mb": 137.383, "state": "S (sleeping)", "threads": 106}, "restart_count": 93, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.946, "sample_count": 1127, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:52:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.895, "last_frame_pushed_at": "2026-07-01T17:52:21+09:00", "last_frame_read_age_sec": 5.156, "last_frame_read_at": "2026-07-01T17:52:21+09:00", "mjpeg": {"age_sec": 4.881, "exists": true, "mtime": "2026-07-01T17:52:21.288496+09:00", "mtime_epoch": 1782895941.2884955, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157233}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:52:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.348, "pushed": 3.348, "read": 3.315}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.804, "count": 99, "max_ms": 12.319, "p95_ms": 10.73}, "gstreamer_push": {"avg_ms": 0.955, "count": 100, "max_ms": 2.863, "p95_ms": 1.808}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.403, "count": 99, "max_ms": 68.993, "p95_ms": 63.349}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.935, "count": 99, "max_ms": 93.391, "p95_ms": 85.714}, "yolo11_640_fire_11class": {"avg_ms": 53.978, "count": 99, "max_ms": 82.263, "p95_ms": 71.518}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.036, "count": 99, "max_ms": 95.742, "p95_ms": 89.219}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.486, "count": 100, "max_ms": 321.911, "p95_ms": 270.797}, "read_interval": {"avg_ms": 251.996, "count": 99, "max_ms": 332.254, "p95_ms": 280.653}, "triton_wait": {"avg_ms": 68.626, "count": 99, "max_ms": 95.755, "p95_ms": 89.233}, "visualize": {"avg_ms": 11.242, "count": 99, "max_ms": 13.517, "p95_ms": 12.177}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 389}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4002.121, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558fd5692880] error while decoding MB 50 30, bytestream -18", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:54:16+09:00", "last_error_summary": "[h264 @ 0x558fd5692880] error while decoding MB 50 30, bytestream -18", "last_exit_at": "2026-07-01T17:52:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.518, "last_started_at": "2026-07-01T17:52:24+09:00", "pid": 13654, "process": {"exists": false, "pid": 13654}, "restart_count": 93, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.703, "sample_count": 1169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46695, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:54:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T17:54:16+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T17:54:16+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T17:54:16.641955+09:00", "mtime_epoch": 1782896056.6419547, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177343}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:54:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.148, "pushed": 4.148, "read": 4.148}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.308, "count": 123, "max_ms": 14.477, "p95_ms": 10.494}, "gstreamer_push": {"avg_ms": 0.905, "count": 124, "max_ms": 2.872, "p95_ms": 0.937}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.132, "count": 123, "max_ms": 67.63, "p95_ms": 61.423}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.833, "count": 123, "max_ms": 87.976, "p95_ms": 84.714}, "yolo11_640_fire_11class": {"avg_ms": 55.068, "count": 123, "max_ms": 77.061, "p95_ms": 70.794}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.652, "count": 123, "max_ms": 89.974, "p95_ms": 86.368}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.979, "count": 124, "max_ms": 291.631, "p95_ms": 263.452}, "read_interval": {"avg_ms": 242.462, "count": 124, "max_ms": 301.265, "p95_ms": 273.856}, "triton_wait": {"avg_ms": 68.838, "count": 123, "max_ms": 89.987, "p95_ms": 86.472}, "visualize": {"avg_ms": 10.816, "count": 123, "max_ms": 13.89, "p95_ms": 12.106}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 389}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4001.508, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:54:16+09:00", "last_error_summary": "[h264 @ 0x558fd5692880] error while decoding MB 50 30, bytestream -18", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:54:16+09:00", "last_error_summary": "[h264 @ 0x558fd5692880] error while decoding MB 50 30, bytestream -18", "last_exit_at": "2026-07-01T17:54:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.702, "last_started_at": "2026-07-01T17:52:24+09:00", "pid": 13654, "process": {"exists": false, "pid": 13654}, "restart_count": 93, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.703, "sample_count": 1169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:54:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:54:16+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-01T17:54:16+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T17:54:16.641955+09:00", "mtime_epoch": 1782896056.6419547, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177343}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:54:16+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.5, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.155, "pushed": 4.155, "read": 4.122}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.302, "count": 124, "max_ms": 14.477, "p95_ms": 10.494}, "gstreamer_push": {"avg_ms": 0.906, "count": 124, "max_ms": 2.872, "p95_ms": 0.937}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.151, "count": 124, "max_ms": 67.63, "p95_ms": 61.423}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.83, "count": 124, "max_ms": 87.976, "p95_ms": 84.714}, "yolo11_640_fire_11class": {"avg_ms": 55.066, "count": 124, "max_ms": 77.061, "p95_ms": 70.794}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.64, "count": 124, "max_ms": 89.974, "p95_ms": 86.368}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.027, "count": 124, "max_ms": 291.631, "p95_ms": 263.452}, "read_interval": {"avg_ms": 242.509, "count": 123, "max_ms": 301.265, "p95_ms": 273.856}, "triton_wait": {"avg_ms": 68.825, "count": 124, "max_ms": 89.987, "p95_ms": 86.472}, "visualize": {"avg_ms": 10.819, "count": 124, "max_ms": 13.89, "p95_ms": 12.106}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.842, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 389}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4001.5, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:54:16+09:00", "last_error_summary": "[h264 @ 0x558fd5692880] error while decoding MB 50 30, bytestream -18", "last_exit_at": "2026-07-01T17:54:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.705, "last_started_at": "2026-07-01T17:52:24+09:00", "pid": 13654, "process": {"exists": false, "pid": 13654}, "restart_count": 93, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 94}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.703, "sample_count": 1169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:54:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:54:16+09:00", "last_frame_read_age_sec": 0.24, "last_frame_read_at": "2026-07-01T17:54:16+09:00", "mjpeg": {"age_sec": 0.232, "exists": true, "mtime": "2026-07-01T17:54:16.641955+09:00", "mtime_epoch": 1782896056.6419547, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177343}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:54:16+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.5, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.155, "pushed": 4.155, "read": 4.121}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.302, "count": 124, "max_ms": 14.477, "p95_ms": 10.494}, "gstreamer_push": {"avg_ms": 0.906, "count": 124, "max_ms": 2.872, "p95_ms": 0.937}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.151, "count": 124, "max_ms": 67.63, "p95_ms": 61.423}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.83, "count": 124, "max_ms": 87.976, "p95_ms": 84.714}, "yolo11_640_fire_11class": {"avg_ms": 55.066, "count": 124, "max_ms": 77.061, "p95_ms": 70.794}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.64, "count": 124, "max_ms": 89.974, "p95_ms": 86.368}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.027, "count": 124, "max_ms": 291.631, "p95_ms": 263.452}, "read_interval": {"avg_ms": 242.509, "count": 123, "max_ms": 301.265, "p95_ms": 273.856}, "triton_wait": {"avg_ms": 68.825, "count": 124, "max_ms": 89.987, "p95_ms": 86.472}, "visualize": {"avg_ms": 10.819, "count": 124, "max_ms": 13.89, "p95_ms": 12.106}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 389}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4106.523, "limit_mb": 8796093022207.996}, "detail": {"pid": 13761, "restart_count": 94, "start_count": 95}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:54:16+09:00", "last_error_summary": "[h264 @ 0x558fd5692880] error while decoding MB 50 30, bytestream -18", "last_exit_at": "2026-07-01T17:54:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:54:19+09:00", "pid": 13761, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.617, "name": "ffmpeg", "pid": 13761, "rss_mb": 138.617, "state": "S (sleeping)", "threads": 106}, "restart_count": 94, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.837, "sample_count": 1150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 46696, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:54:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.525, "last_frame_pushed_at": "2026-07-01T17:54:16+09:00", "last_frame_read_age_sec": 4.758, "last_frame_read_at": "2026-07-01T17:54:16+09:00", "mjpeg": {"age_sec": 4.506, "exists": true, "mtime": "2026-07-01T17:54:16.885958+09:00", "mtime_epoch": 1782896056.885958, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175190}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:54:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.539, "pushed": 3.539, "read": 3.505}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.397, "count": 105, "max_ms": 14.477, "p95_ms": 10.521}, "gstreamer_push": {"avg_ms": 0.902, "count": 106, "max_ms": 2.872, "p95_ms": 0.937}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.963, "count": 105, "max_ms": 67.63, "p95_ms": 61.497}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.739, "count": 105, "max_ms": 87.355, "p95_ms": 84.714}, "yolo11_640_fire_11class": {"avg_ms": 55.122, "count": 105, "max_ms": 77.061, "p95_ms": 71.337}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.775, "count": 105, "max_ms": 89.974, "p95_ms": 86.459}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.757, "count": 106, "max_ms": 280.896, "p95_ms": 263.452}, "read_interval": {"avg_ms": 242.237, "count": 105, "max_ms": 292.088, "p95_ms": 273.856}, "triton_wait": {"avg_ms": 68.869, "count": 105, "max_ms": 89.987, "p95_ms": 86.472}, "visualize": {"avg_ms": 10.788, "count": 105, "max_ms": 12.67, "p95_ms": 12.106}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 393}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3995.48, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a66224ca00] error while decoding MB 24 23, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:56:13+09:00", "last_error_summary": "[h264 @ 0x55a66224ca00] error while decoding MB 24 23, bytestream -5", "last_exit_at": "2026-07-01T17:54:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.961, "last_started_at": "2026-07-01T17:54:19+09:00", "pid": 13761, "process": {"exists": false, "pid": 13761}, "restart_count": 94, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.951, "sample_count": 1189, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47160, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:56:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T17:56:13+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T17:56:13+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T17:56:13.311495+09:00", "mtime_epoch": 1782896173.3114953, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185544}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:56:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 878.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.134, "pushed": 4.134, "read": 4.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.403, "count": 123, "max_ms": 13.416, "p95_ms": 10.094}, "gstreamer_push": {"avg_ms": 0.923, "count": 124, "max_ms": 2.917, "p95_ms": 1.758}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.734, "count": 123, "max_ms": 74.486, "p95_ms": 60.687}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.395, "count": 123, "max_ms": 87.442, "p95_ms": 80.726}, "yolo11_640_fire_11class": {"avg_ms": 53.451, "count": 123, "max_ms": 74.473, "p95_ms": 68.551}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.945, "count": 123, "max_ms": 88.774, "p95_ms": 85.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.758, "count": 124, "max_ms": 294.915, "p95_ms": 263.657}, "read_interval": {"avg_ms": 243.287, "count": 124, "max_ms": 306.153, "p95_ms": 272.915}, "triton_wait": {"avg_ms": 67.078, "count": 123, "max_ms": 88.786, "p95_ms": 85.097}, "visualize": {"avg_ms": 11.026, "count": 123, "max_ms": 12.757, "p95_ms": 12.057}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 393}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3994.68, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:56:13+09:00", "last_error_summary": "[h264 @ 0x55a66224ca00] error while decoding MB 24 23, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:56:13+09:00", "last_error_summary": "[h264 @ 0x55a66224ca00] error while decoding MB 24 23, bytestream -5", "last_exit_at": "2026-07-01T17:56:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.142, "last_started_at": "2026-07-01T17:54:19+09:00", "pid": 13761, "process": {"exists": false, "pid": 13761}, "restart_count": 94, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.951, "sample_count": 1189, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:56:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:56:13+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T17:56:13+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T17:56:13.311495+09:00", "mtime_epoch": 1782896173.3114953, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185544}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:56:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 878.465, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.144, "pushed": 4.144, "read": 4.111}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.402, "count": 123, "max_ms": 13.416, "p95_ms": 10.094}, "gstreamer_push": {"avg_ms": 0.924, "count": 124, "max_ms": 2.917, "p95_ms": 1.758}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.657, "count": 123, "max_ms": 74.486, "p95_ms": 60.687}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.304, "count": 123, "max_ms": 87.442, "p95_ms": 80.726}, "yolo11_640_fire_11class": {"avg_ms": 53.354, "count": 123, "max_ms": 74.473, "p95_ms": 68.551}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.855, "count": 123, "max_ms": 88.774, "p95_ms": 85.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.765, "count": 124, "max_ms": 294.915, "p95_ms": 263.657}, "read_interval": {"avg_ms": 243.328, "count": 123, "max_ms": 306.153, "p95_ms": 272.915}, "triton_wait": {"avg_ms": 66.989, "count": 123, "max_ms": 88.786, "p95_ms": 85.097}, "visualize": {"avg_ms": 11.031, "count": 123, "max_ms": 12.757, "p95_ms": 12.057}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 393}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3994.672, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:56:13+09:00", "last_error_summary": "[h264 @ 0x55a66224ca00] error while decoding MB 24 23, bytestream -5", "last_exit_at": "2026-07-01T17:56:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.146, "last_started_at": "2026-07-01T17:54:19+09:00", "pid": 13761, "process": {"exists": false, "pid": 13761}, "restart_count": 94, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 95}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.951, "sample_count": 1189, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:56:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:56:13+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T17:56:13+09:00", "mjpeg": {"age_sec": 0.225, "exists": true, "mtime": "2026-07-01T17:56:13.311495+09:00", "mtime_epoch": 1782896173.3114953, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185544}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:56:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 878.465, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.144, "pushed": 4.144, "read": 4.11}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.402, "count": 123, "max_ms": 13.416, "p95_ms": 10.094}, "gstreamer_push": {"avg_ms": 0.924, "count": 124, "max_ms": 2.917, "p95_ms": 1.758}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.657, "count": 123, "max_ms": 74.486, "p95_ms": 60.687}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.304, "count": 123, "max_ms": 87.442, "p95_ms": 80.726}, "yolo11_640_fire_11class": {"avg_ms": 53.354, "count": 123, "max_ms": 74.473, "p95_ms": 68.551}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.855, "count": 123, "max_ms": 88.774, "p95_ms": 85.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.765, "count": 124, "max_ms": 294.915, "p95_ms": 263.657}, "read_interval": {"avg_ms": 243.328, "count": 123, "max_ms": 306.153, "p95_ms": 272.915}, "triton_wait": {"avg_ms": 66.989, "count": 123, "max_ms": 88.786, "p95_ms": 85.097}, "visualize": {"avg_ms": 11.031, "count": 123, "max_ms": 12.757, "p95_ms": 12.057}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 393}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4097.156, "limit_mb": 8796093022207.996}, "detail": {"pid": 13868, "restart_count": 95, "start_count": 96}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:56:13+09:00", "last_error_summary": "[h264 @ 0x55a66224ca00] error while decoding MB 24 23, bytestream -5", "last_exit_at": "2026-07-01T17:56:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:56:16+09:00", "pid": 13868, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.422, "name": "ffmpeg", "pid": 13868, "rss_mb": 136.422, "state": "S (sleeping)", "threads": 106}, "restart_count": 95, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 96}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.957, "sample_count": 1170, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:56:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.47, "last_frame_pushed_at": "2026-07-01T17:56:13+09:00", "last_frame_read_age_sec": 4.702, "last_frame_read_at": "2026-07-01T17:56:13+09:00", "mjpeg": {"age_sec": 4.454, "exists": true, "mtime": "2026-07-01T17:56:13.547498+09:00", "mtime_epoch": 1782896173.5474985, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185102}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:56:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 878.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.539, "pushed": 3.539, "read": 3.506}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.44, "count": 105, "max_ms": 13.416, "p95_ms": 11.108}, "gstreamer_push": {"avg_ms": 0.906, "count": 106, "max_ms": 1.926, "p95_ms": 0.965}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.782, "count": 105, "max_ms": 74.486, "p95_ms": 60.87}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.486, "count": 105, "max_ms": 87.442, "p95_ms": 80.726}, "yolo11_640_fire_11class": {"avg_ms": 53.539, "count": 105, "max_ms": 74.473, "p95_ms": 68.551}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.103, "count": 105, "max_ms": 88.774, "p95_ms": 85.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.083, "count": 106, "max_ms": 294.915, "p95_ms": 263.657}, "read_interval": {"avg_ms": 242.606, "count": 105, "max_ms": 306.153, "p95_ms": 272.915}, "triton_wait": {"avg_ms": 67.243, "count": 105, "max_ms": 88.786, "p95_ms": 85.097}, "visualize": {"avg_ms": 10.951, "count": 105, "max_ms": 12.757, "p95_ms": 11.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 396}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4006.48, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562727fe6740] cabac decode of qscale diff failed at 79 16 [h264 @ 0x562727fe6740] error while decoding MB 79 16, bytestream -10", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 85, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:58:07+09:00", "last_error_summary": "[h264 @ 0x562727fe6740] cabac decode of qscale diff failed at 79 16 [h264 @ 0x562727fe6740] error while decoding MB 79 16, bytestream -10", "last_exit_at": "2026-07-01T17:56:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.525, "last_started_at": "2026-07-01T17:56:16+09:00", "pid": 13868, "process": {"exists": false, "pid": 13868}, "restart_count": 95, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 96}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.446, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47594, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:58:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T17:58:07+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T17:58:07+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T17:58:07.481053+09:00", "mtime_epoch": 1782896287.4810534, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190910}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:58:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.84, "pushed": 3.84, "read": 3.84}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.91, "count": 114, "max_ms": 17.984, "p95_ms": 10.776}, "gstreamer_push": {"avg_ms": 0.936, "count": 115, "max_ms": 2.466, "p95_ms": 1.805}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.761, "count": 114, "max_ms": 69.813, "p95_ms": 66.762}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.223, "count": 114, "max_ms": 89.11, "p95_ms": 86.619}, "yolo11_640_fire_11class": {"avg_ms": 56.048, "count": 114, "max_ms": 81.955, "p95_ms": 72.857}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.811, "count": 114, "max_ms": 94.028, "p95_ms": 90.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 252.659, "count": 115, "max_ms": 347.397, "p95_ms": 290.815}, "read_interval": {"avg_ms": 262.383, "count": 115, "max_ms": 357.336, "p95_ms": 300.031}, "triton_wait": {"avg_ms": 71.097, "count": 114, "max_ms": 94.041, "p95_ms": 90.404}, "visualize": {"avg_ms": 11.747, "count": 114, "max_ms": 18.072, "p95_ms": 13.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 396}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4005.598, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T17:58:07+09:00", "last_error_summary": "[h264 @ 0x562727fe6740] cabac decode of qscale diff failed at 79 16 [h264 @ 0x562727fe6740] error while decoding MB 79 16, bytestream -10", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 85, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:58:07+09:00", "last_error_summary": "[h264 @ 0x562727fe6740] cabac decode of qscale diff failed at 79 16 [h264 @ 0x562727fe6740] error while decoding MB 79 16, bytestream -10", "last_exit_at": "2026-07-01T17:58:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.731, "last_started_at": "2026-07-01T17:56:16+09:00", "pid": 13868, "process": {"exists": false, "pid": 13868}, "restart_count": 95, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 96}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.446, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47595, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:58:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T17:58:07+09:00", "last_frame_read_age_sec": 0.258, "last_frame_read_at": "2026-07-01T17:58:07+09:00", "mjpeg": {"age_sec": 0.25, "exists": true, "mtime": "2026-07-01T17:58:07.481053+09:00", "mtime_epoch": 1782896287.4810534, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190910}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:58:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.426, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.848, "pushed": 3.848, "read": 3.815}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.92, "count": 114, "max_ms": 17.984, "p95_ms": 10.829}, "gstreamer_push": {"avg_ms": 0.936, "count": 115, "max_ms": 2.466, "p95_ms": 1.805}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.507, "count": 114, "max_ms": 69.813, "p95_ms": 66.218}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.039, "count": 114, "max_ms": 89.11, "p95_ms": 86.619}, "yolo11_640_fire_11class": {"avg_ms": 55.871, "count": 114, "max_ms": 81.955, "p95_ms": 72.857}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.579, "count": 114, "max_ms": 94.028, "p95_ms": 90.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 252.474, "count": 115, "max_ms": 347.397, "p95_ms": 290.815}, "read_interval": {"avg_ms": 262.197, "count": 114, "max_ms": 357.336, "p95_ms": 300.031}, "triton_wait": {"avg_ms": 70.872, "count": 114, "max_ms": 94.041, "p95_ms": 90.404}, "visualize": {"avg_ms": 11.805, "count": 114, "max_ms": 18.476, "p95_ms": 14.658}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 396}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4005.676, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 85, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:58:07+09:00", "last_error_summary": "[h264 @ 0x562727fe6740] cabac decode of qscale diff failed at 79 16 [h264 @ 0x562727fe6740] error while decoding MB 79 16, bytestream -10", "last_exit_at": "2026-07-01T17:58:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.734, "last_started_at": "2026-07-01T17:56:16+09:00", "pid": 13868, "process": {"exists": false, "pid": 13868}, "restart_count": 95, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 96}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.446, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47595, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:58:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T17:58:07+09:00", "last_frame_read_age_sec": 0.261, "last_frame_read_at": "2026-07-01T17:58:07+09:00", "mjpeg": {"age_sec": 0.253, "exists": true, "mtime": "2026-07-01T17:58:07.481053+09:00", "mtime_epoch": 1782896287.4810534, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190910}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:58:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.426, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.848, "pushed": 3.848, "read": 3.815}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.92, "count": 114, "max_ms": 17.984, "p95_ms": 10.829}, "gstreamer_push": {"avg_ms": 0.936, "count": 115, "max_ms": 2.466, "p95_ms": 1.805}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.507, "count": 114, "max_ms": 69.813, "p95_ms": 66.218}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.039, "count": 114, "max_ms": 89.11, "p95_ms": 86.619}, "yolo11_640_fire_11class": {"avg_ms": 55.871, "count": 114, "max_ms": 81.955, "p95_ms": 72.857}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.579, "count": 114, "max_ms": 94.028, "p95_ms": 90.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 252.474, "count": 115, "max_ms": 347.397, "p95_ms": 290.815}, "read_interval": {"avg_ms": 262.197, "count": 114, "max_ms": 357.336, "p95_ms": 300.031}, "triton_wait": {"avg_ms": 70.872, "count": 114, "max_ms": 94.041, "p95_ms": 90.404}, "visualize": {"avg_ms": 11.805, "count": 114, "max_ms": 18.476, "p95_ms": 14.658}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 396}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4113.691, "limit_mb": 8796093022207.996}, "detail": {"pid": 13975, "restart_count": 96, "start_count": 97}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 85, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T17:58:07+09:00", "last_error_summary": "[h264 @ 0x562727fe6740] cabac decode of qscale diff failed at 79 16 [h264 @ 0x562727fe6740] error while decoding MB 79 16, bytestream -10", "last_exit_at": "2026-07-01T17:58:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T17:58:10+09:00", "pid": 13975, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.883, "name": "ffmpeg", "pid": 13975, "rss_mb": 138.883, "state": "S (sleeping)", "threads": 106}, "restart_count": 96, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 97}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.446, "sample_count": 1165, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 47595, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T17:58:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.983, "last_frame_pushed_at": "2026-07-01T17:58:07+09:00", "last_frame_read_age_sec": 5.238, "last_frame_read_at": "2026-07-01T17:58:07+09:00", "mjpeg": {"age_sec": 4.966, "exists": true, "mtime": "2026-07-01T17:58:07.745057+09:00", "mtime_epoch": 1782896287.745057, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190422}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T17:58:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.236, "pushed": 3.236, "read": 3.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.821, "count": 96, "max_ms": 12.828, "p95_ms": 10.776}, "gstreamer_push": {"avg_ms": 0.936, "count": 97, "max_ms": 2.466, "p95_ms": 1.808}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.629, "count": 96, "max_ms": 69.813, "p95_ms": 66.035}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.588, "count": 96, "max_ms": 87.312, "p95_ms": 86.619}, "yolo11_640_fire_11class": {"avg_ms": 55.056, "count": 96, "max_ms": 73.749, "p95_ms": 71.818}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.858, "count": 96, "max_ms": 94.028, "p95_ms": 89.535}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.55, "count": 97, "max_ms": 347.397, "p95_ms": 276.999}, "read_interval": {"avg_ms": 260.238, "count": 96, "max_ms": 357.336, "p95_ms": 287.324}, "triton_wait": {"avg_ms": 70.194, "count": 96, "max_ms": 94.041, "p95_ms": 89.546}, "visualize": {"avg_ms": 11.744, "count": 96, "max_ms": 18.476, "p95_ms": 13.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4004.59, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560eea442580] error while decoding MB 23 17, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:00:01+09:00", "last_error_summary": "[h264 @ 0x560eea442580] error while decoding MB 23 17, bytestream -5", "last_exit_at": "2026-07-01T17:58:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.88, "last_started_at": "2026-07-01T17:58:10+09:00", "pid": 13975, "process": {"exists": false, "pid": 13975}, "restart_count": 96, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 97}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.361, "sample_count": 1158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48023, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:00:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T18:00:01+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T18:00:01+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T18:00:01.546650+09:00", "mtime_epoch": 1782896401.5466497, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:00:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.062, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.912, "pushed": 3.912, "read": 3.912}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.769, "count": 116, "max_ms": 11.046, "p95_ms": 10.526}, "gstreamer_push": {"avg_ms": 0.931, "count": 117, "max_ms": 2.972, "p95_ms": 1.766}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.752, "count": 116, "max_ms": 72.667, "p95_ms": 63.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.91, "count": 116, "max_ms": 95.481, "p95_ms": 88.128}, "yolo11_640_fire_11class": {"avg_ms": 54.598, "count": 116, "max_ms": 78.191, "p95_ms": 74.803}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.792, "count": 116, "max_ms": 95.494, "p95_ms": 91.748}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.468, "count": 117, "max_ms": 318.429, "p95_ms": 277.467}, "read_interval": {"avg_ms": 257.057, "count": 117, "max_ms": 329.346, "p95_ms": 285.297}, "triton_wait": {"avg_ms": 69.479, "count": 116, "max_ms": 95.504, "p95_ms": 91.761}, "visualize": {"avg_ms": 11.51, "count": 116, "max_ms": 12.591, "p95_ms": 12.287}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3998.566, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:00:01+09:00", "last_error_summary": "[h264 @ 0x560eea442580] error while decoding MB 23 17, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:00:01+09:00", "last_error_summary": "[h264 @ 0x560eea442580] error while decoding MB 23 17, bytestream -5", "last_exit_at": "2026-07-01T18:00:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.058, "last_started_at": "2026-07-01T17:58:10+09:00", "pid": 13975, "process": {"exists": false, "pid": 13975}, "restart_count": 96, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 97}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.361, "sample_count": 1159, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48024, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:00:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:00:01+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T18:00:01+09:00", "mjpeg": {"age_sec": 0.221, "exists": true, "mtime": "2026-07-01T18:00:01.546650+09:00", "mtime_epoch": 1782896401.5466497, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:00:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.062, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.92, "pushed": 3.92, "read": 3.886}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.764, "count": 117, "max_ms": 11.046, "p95_ms": 10.526}, "gstreamer_push": {"avg_ms": 0.93, "count": 117, "max_ms": 2.972, "p95_ms": 1.766}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.731, "count": 117, "max_ms": 72.667, "p95_ms": 63.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.861, "count": 117, "max_ms": 95.481, "p95_ms": 88.128}, "yolo11_640_fire_11class": {"avg_ms": 54.604, "count": 117, "max_ms": 78.191, "p95_ms": 74.803}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.764, "count": 117, "max_ms": 95.494, "p95_ms": 91.748}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.502, "count": 117, "max_ms": 318.429, "p95_ms": 277.467}, "read_interval": {"avg_ms": 257.286, "count": 116, "max_ms": 329.346, "p95_ms": 285.297}, "triton_wait": {"avg_ms": 69.446, "count": 117, "max_ms": 95.504, "p95_ms": 91.761}, "visualize": {"avg_ms": 11.501, "count": 117, "max_ms": 12.591, "p95_ms": 12.287}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.847, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3998.559, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:00:01+09:00", "last_error_summary": "[h264 @ 0x560eea442580] error while decoding MB 23 17, bytestream -5", "last_exit_at": "2026-07-01T18:00:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.061, "last_started_at": "2026-07-01T17:58:10+09:00", "pid": 13975, "process": {"exists": false, "pid": 13975}, "restart_count": 96, "restart_count_per_hour": 31, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 97}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.361, "sample_count": 1159, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48024, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:00:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:00:01+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-01T18:00:01+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T18:00:01.546650+09:00", "mtime_epoch": 1782896401.5466497, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:00:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.062, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.92, "pushed": 3.92, "read": 3.886}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.764, "count": 117, "max_ms": 11.046, "p95_ms": 10.526}, "gstreamer_push": {"avg_ms": 0.93, "count": 117, "max_ms": 2.972, "p95_ms": 1.766}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.731, "count": 117, "max_ms": 72.667, "p95_ms": 63.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.861, "count": 117, "max_ms": 95.481, "p95_ms": 88.128}, "yolo11_640_fire_11class": {"avg_ms": 54.604, "count": 117, "max_ms": 78.191, "p95_ms": 74.803}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.764, "count": 117, "max_ms": 95.494, "p95_ms": 91.748}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.502, "count": 117, "max_ms": 318.429, "p95_ms": 277.467}, "read_interval": {"avg_ms": 257.286, "count": 116, "max_ms": 329.346, "p95_ms": 285.297}, "triton_wait": {"avg_ms": 69.446, "count": 117, "max_ms": 95.504, "p95_ms": 91.761}, "visualize": {"avg_ms": 11.501, "count": 117, "max_ms": 12.591, "p95_ms": 12.287}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.851, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4108.863, "limit_mb": 8796093022207.996}, "detail": {"pid": 14082, "restart_count": 97, "start_count": 98}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:00:01+09:00", "last_error_summary": "[h264 @ 0x560eea442580] error while decoding MB 23 17, bytestream -5", "last_exit_at": "2026-07-01T18:00:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:00:04+09:00", "pid": 14082, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.891, "name": "ffmpeg", "pid": 14082, "rss_mb": 138.891, "state": "S (sleeping)", "threads": 106}, "restart_count": 97, "restart_count_per_hour": 32, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 98}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.361, "sample_count": 1140, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48024, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:00:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.642, "last_frame_pushed_at": "2026-07-01T18:00:01+09:00", "last_frame_read_age_sec": 4.868, "last_frame_read_at": "2026-07-01T18:00:01+09:00", "mjpeg": {"age_sec": 4.624, "exists": true, "mtime": "2026-07-01T18:00:01.782653+09:00", "mtime_epoch": 1782896401.782653, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188932}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:00:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.062, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.268}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.882, "count": 98, "max_ms": 11.046, "p95_ms": 10.563}, "gstreamer_push": {"avg_ms": 0.924, "count": 99, "max_ms": 2.972, "p95_ms": 1.758}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.387, "count": 98, "max_ms": 72.667, "p95_ms": 63.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.005, "count": 98, "max_ms": 95.481, "p95_ms": 88.755}, "yolo11_640_fire_11class": {"avg_ms": 54.661, "count": 98, "max_ms": 78.191, "p95_ms": 74.847}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.909, "count": 98, "max_ms": 95.494, "p95_ms": 91.748}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.472, "count": 99, "max_ms": 318.429, "p95_ms": 277.467}, "read_interval": {"avg_ms": 259.115, "count": 98, "max_ms": 329.346, "p95_ms": 285.297}, "triton_wait": {"avg_ms": 69.703, "count": 98, "max_ms": 95.504, "p95_ms": 91.761}, "visualize": {"avg_ms": 11.577, "count": 98, "max_ms": 12.591, "p95_ms": 12.373}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 404}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4006.129, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 0, "rtsp_timeout_count": 2, "sample_message": "[in#0/rtsp @ 0x5567cc014e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5567cc014e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:02:43+09:00", "last_error_summary": "[in#0/rtsp @ 0x5567cc014e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5567cc014e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T18:00:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.836, "last_started_at": "2026-07-01T18:00:04+09:00", "pid": 14082, "process": {"exists": false, "pid": 14082}, "restart_count": 97, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 98}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.006, "sample_count": 1086, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48587, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:02:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T18:02:43+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T18:02:43+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T18:02:43.200965+09:00", "mtime_epoch": 1782896563.2009652, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195308}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:02:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.047, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 51, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.71, "pushed": 1.71, "read": 1.71}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.082, "count": 51, "max_ms": 9.842, "p95_ms": 9.69}, "gstreamer_push": {"avg_ms": 0.91, "count": 51, "max_ms": 1.968, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.649, "count": 51, "max_ms": 70.435, "p95_ms": 61.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.348, "count": 51, "max_ms": 89.4, "p95_ms": 88.349}, "yolo11_640_fire_11class": {"avg_ms": 55.101, "count": 51, "max_ms": 73.643, "p95_ms": 72.535}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.393, "count": 51, "max_ms": 92.202, "p95_ms": 90.416}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.825, "count": 51, "max_ms": 301.494, "p95_ms": 253.766}, "read_interval": {"avg_ms": 588.237, "count": 51, "max_ms": 10131.675, "p95_ms": 311.877}, "triton_wait": {"avg_ms": 69.677, "count": 51, "max_ms": 92.214, "p95_ms": 90.428}, "visualize": {"avg_ms": 10.808, "count": 51, "max_ms": 11.733, "p95_ms": 11.461}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.826, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 404}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4005.18, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:02:43+09:00", "last_error_summary": "[in#0/rtsp @ 0x5567cc014e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5567cc014e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:02:43+09:00", "last_error_summary": "[in#0/rtsp @ 0x5567cc014e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5567cc014e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T18:02:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.006, "last_started_at": "2026-07-01T18:00:04+09:00", "pid": 14082, "process": {"exists": false, "pid": 14082}, "restart_count": 97, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 98}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.006, "sample_count": 1087, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:02:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-01T18:02:43+09:00", "last_frame_read_age_sec": 0.221, "last_frame_read_at": "2026-07-01T18:02:43+09:00", "mjpeg": {"age_sec": 0.211, "exists": true, "mtime": "2026-07-01T18:02:43.200965+09:00", "mtime_epoch": 1782896563.2009652, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195308}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:02:43+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.047, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 51, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.734, "pushed": 1.734, "read": 1.7}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.1, "count": 51, "max_ms": 9.842, "p95_ms": 9.697}, "gstreamer_push": {"avg_ms": 0.909, "count": 52, "max_ms": 1.968, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.51, "count": 51, "max_ms": 70.435, "p95_ms": 61.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.128, "count": 51, "max_ms": 89.4, "p95_ms": 88.349}, "yolo11_640_fire_11class": {"avg_ms": 54.847, "count": 51, "max_ms": 73.643, "p95_ms": 72.535}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.158, "count": 51, "max_ms": 92.202, "p95_ms": 90.416}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.532, "count": 52, "max_ms": 301.494, "p95_ms": 253.766}, "read_interval": {"avg_ms": 588.237, "count": 51, "max_ms": 10131.675, "p95_ms": 311.877}, "triton_wait": {"avg_ms": 69.442, "count": 51, "max_ms": 92.214, "p95_ms": 90.428}, "visualize": {"avg_ms": 10.849, "count": 51, "max_ms": 12.714, "p95_ms": 11.651}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 404}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4005.172, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:02:43+09:00", "last_error_summary": "[in#0/rtsp @ 0x5567cc014e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5567cc014e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T18:02:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.009, "last_started_at": "2026-07-01T18:00:04+09:00", "pid": 14082, "process": {"exists": false, "pid": 14082}, "restart_count": 97, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 98}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.006, "sample_count": 1087, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:02:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T18:02:43+09:00", "last_frame_read_age_sec": 0.224, "last_frame_read_at": "2026-07-01T18:02:43+09:00", "mjpeg": {"age_sec": 0.214, "exists": true, "mtime": "2026-07-01T18:02:43.200965+09:00", "mtime_epoch": 1782896563.2009652, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195308}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:02:43+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.047, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 51, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.733, "pushed": 1.733, "read": 1.7}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.1, "count": 51, "max_ms": 9.842, "p95_ms": 9.697}, "gstreamer_push": {"avg_ms": 0.909, "count": 52, "max_ms": 1.968, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.51, "count": 51, "max_ms": 70.435, "p95_ms": 61.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.128, "count": 51, "max_ms": 89.4, "p95_ms": 88.349}, "yolo11_640_fire_11class": {"avg_ms": 54.847, "count": 51, "max_ms": 73.643, "p95_ms": 72.535}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.158, "count": 51, "max_ms": 92.202, "p95_ms": 90.416}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.532, "count": 52, "max_ms": 301.494, "p95_ms": 253.766}, "read_interval": {"avg_ms": 588.237, "count": 51, "max_ms": 10131.675, "p95_ms": 311.877}, "triton_wait": {"avg_ms": 69.442, "count": 51, "max_ms": 92.214, "p95_ms": 90.428}, "visualize": {"avg_ms": 10.849, "count": 51, "max_ms": 12.714, "p95_ms": 11.651}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 404}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4108.562, "limit_mb": 8796093022207.996}, "detail": {"pid": 14189, "restart_count": 98, "start_count": 99}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:02:43+09:00", "last_error_summary": "[in#0/rtsp @ 0x5567cc014e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5567cc014e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T18:02:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:02:46+09:00", "pid": 14189, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.684, "name": "ffmpeg", "pid": 14189, "rss_mb": 136.684, "state": "R (running)", "threads": 106}, "restart_count": 98, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 99}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.006, "sample_count": 1068, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48588, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:02:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.806, "last_frame_pushed_at": "2026-07-01T18:02:43+09:00", "last_frame_read_age_sec": 5.025, "last_frame_read_at": "2026-07-01T18:02:43+09:00", "mjpeg": {"age_sec": 4.791, "exists": true, "mtime": "2026-07-01T18:02:43.424968+09:00", "mtime_epoch": 1782896563.4249682, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196078}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:02:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.047, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 32, "pushed_frames": 32, "read_frames": 31, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.069, "pushed": 1.069, "read": 1.036}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.147, "count": 31, "max_ms": 9.722, "p95_ms": 9.697}, "gstreamer_push": {"avg_ms": 0.924, "count": 32, "max_ms": 1.968, "p95_ms": 1.877}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.156, "count": 31, "max_ms": 70.435, "p95_ms": 61.399}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.761, "count": 31, "max_ms": 89.4, "p95_ms": 85.971}, "yolo11_640_fire_11class": {"avg_ms": 54.945, "count": 31, "max_ms": 73.643, "p95_ms": 72.541}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.885, "count": 31, "max_ms": 92.202, "p95_ms": 90.416}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.823, "count": 32, "max_ms": 284.333, "p95_ms": 253.766}, "read_interval": {"avg_ms": 810.839, "count": 31, "max_ms": 10131.675, "p95_ms": 7987.668}, "triton_wait": {"avg_ms": 69.113, "count": 31, "max_ms": 92.214, "p95_ms": 90.428}, "visualize": {"avg_ms": 10.897, "count": 31, "max_ms": 12.714, "p95_ms": 11.733}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 408}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4009.453, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562b13100840] error while decoding MB 36 20, bytestream -34", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:04:38+09:00", "last_error_summary": "[h264 @ 0x562b13100840] error while decoding MB 36 20, bytestream -34", "last_exit_at": "2026-07-01T18:02:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.062, "last_started_at": "2026-07-01T18:02:46+09:00", "pid": 14189, "process": {"exists": false, "pid": 14189}, "restart_count": 98, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 99}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.829, "sample_count": 1116, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49049, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:04:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T18:04:38+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T18:04:38+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T18:04:38.234642+09:00", "mtime_epoch": 1782896678.234642, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187849}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:04:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.395, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.089, "pushed": 4.089, "read": 4.089}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.375, "count": 121, "max_ms": 13.871, "p95_ms": 10.201}, "gstreamer_push": {"avg_ms": 0.92, "count": 122, "max_ms": 2.698, "p95_ms": 1.054}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.166, "count": 121, "max_ms": 71.093, "p95_ms": 61.272}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.198, "count": 121, "max_ms": 98.672, "p95_ms": 87.245}, "yolo11_640_fire_11class": {"avg_ms": 53.261, "count": 121, "max_ms": 76.364, "p95_ms": 71.06}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.243, "count": 121, "max_ms": 97.595, "p95_ms": 87.895}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.859, "count": 122, "max_ms": 309.594, "p95_ms": 282.629}, "read_interval": {"avg_ms": 246.492, "count": 122, "max_ms": 318.65, "p95_ms": 293.525}, "triton_wait": {"avg_ms": 67.852, "count": 121, "max_ms": 98.685, "p95_ms": 88.025}, "visualize": {"avg_ms": 11.05, "count": 121, "max_ms": 13.315, "p95_ms": 12.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.836, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 408}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4008.785, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:04:38+09:00", "last_error_summary": "[h264 @ 0x562b13100840] error while decoding MB 36 20, bytestream -34", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:04:38+09:00", "last_error_summary": "[h264 @ 0x562b13100840] error while decoding MB 36 20, bytestream -34", "last_exit_at": "2026-07-01T18:04:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.237, "last_started_at": "2026-07-01T18:02:46+09:00", "pid": 14189, "process": {"exists": false, "pid": 14189}, "restart_count": 98, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 99}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.829, "sample_count": 1116, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:04:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:04:38+09:00", "last_frame_read_age_sec": 0.229, "last_frame_read_at": "2026-07-01T18:04:38+09:00", "mjpeg": {"age_sec": 0.218, "exists": true, "mtime": "2026-07-01T18:04:38.234642+09:00", "mtime_epoch": 1782896678.234642, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187849}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:04:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.395, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.067, "pushed": 4.067, "read": 4.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.398, "count": 122, "max_ms": 13.871, "p95_ms": 10.833}, "gstreamer_push": {"avg_ms": 0.905, "count": 122, "max_ms": 1.851, "p95_ms": 1.011}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.084, "count": 122, "max_ms": 71.093, "p95_ms": 61.272}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.121, "count": 122, "max_ms": 98.672, "p95_ms": 87.245}, "yolo11_640_fire_11class": {"avg_ms": 53.192, "count": 122, "max_ms": 76.364, "p95_ms": 71.06}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.172, "count": 122, "max_ms": 97.595, "p95_ms": 87.895}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.33, "count": 122, "max_ms": 309.594, "p95_ms": 277.581}, "read_interval": {"avg_ms": 246.492, "count": 122, "max_ms": 318.65, "p95_ms": 293.525}, "triton_wait": {"avg_ms": 67.777, "count": 122, "max_ms": 98.685, "p95_ms": 88.025}, "visualize": {"avg_ms": 11.058, "count": 122, "max_ms": 13.315, "p95_ms": 12.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 408}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4008.727, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:04:38+09:00", "last_error_summary": "[h264 @ 0x562b13100840] error while decoding MB 36 20, bytestream -34", "last_exit_at": "2026-07-01T18:04:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.241, "last_started_at": "2026-07-01T18:02:46+09:00", "pid": 14189, "process": {"exists": false, "pid": 14189}, "restart_count": 98, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 99}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.829, "sample_count": 1116, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:04:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:04:38+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T18:04:38+09:00", "mjpeg": {"age_sec": 0.221, "exists": true, "mtime": "2026-07-01T18:04:38.234642+09:00", "mtime_epoch": 1782896678.234642, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187849}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:04:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.395, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.106, "pushed": 4.106, "read": 4.072}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.398, "count": 122, "max_ms": 13.871, "p95_ms": 10.833}, "gstreamer_push": {"avg_ms": 0.905, "count": 122, "max_ms": 1.851, "p95_ms": 1.011}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.084, "count": 122, "max_ms": 71.093, "p95_ms": 61.272}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.121, "count": 122, "max_ms": 98.672, "p95_ms": 87.245}, "yolo11_640_fire_11class": {"avg_ms": 53.192, "count": 122, "max_ms": 76.364, "p95_ms": 71.06}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.172, "count": 122, "max_ms": 97.595, "p95_ms": 87.895}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.33, "count": 122, "max_ms": 309.594, "p95_ms": 277.581}, "read_interval": {"avg_ms": 246.036, "count": 121, "max_ms": 318.65, "p95_ms": 286.68}, "triton_wait": {"avg_ms": 67.777, "count": 122, "max_ms": 98.685, "p95_ms": 88.025}, "visualize": {"avg_ms": 11.058, "count": 122, "max_ms": 13.315, "p95_ms": 12.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.712, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 408}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4108.609, "limit_mb": 8796093022207.996}, "detail": {"pid": 14296, "restart_count": 99, "start_count": 100}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:04:38+09:00", "last_error_summary": "[h264 @ 0x562b13100840] error while decoding MB 36 20, bytestream -34", "last_exit_at": "2026-07-01T18:04:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:04:41+09:00", "pid": 14296, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.68, "name": "ffmpeg", "pid": 14296, "rss_mb": 136.68, "state": "S (sleeping)", "threads": 106}, "restart_count": 99, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 100}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.797, "sample_count": 1098, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:04:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.681, "last_frame_pushed_at": "2026-07-01T18:04:38+09:00", "last_frame_read_age_sec": 4.908, "last_frame_read_at": "2026-07-01T18:04:38+09:00", "mjpeg": {"age_sec": 4.664, "exists": true, "mtime": "2026-07-01T18:04:38.466645+09:00", "mtime_epoch": 1782896678.4666452, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:04:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 887.395, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.447, "pushed": 3.447, "read": 3.414}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.43, "count": 102, "max_ms": 13.871, "p95_ms": 11.037}, "gstreamer_push": {"avg_ms": 0.903, "count": 103, "max_ms": 1.851, "p95_ms": 1.011}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.416, "count": 102, "max_ms": 71.093, "p95_ms": 61.272}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.245, "count": 102, "max_ms": 98.672, "p95_ms": 86.938}, "yolo11_640_fire_11class": {"avg_ms": 53.286, "count": 102, "max_ms": 72.924, "p95_ms": 71.06}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.327, "count": 102, "max_ms": 97.595, "p95_ms": 87.895}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.494, "count": 103, "max_ms": 309.594, "p95_ms": 277.581}, "read_interval": {"avg_ms": 247.213, "count": 102, "max_ms": 318.65, "p95_ms": 286.68}, "triton_wait": {"avg_ms": 67.931, "count": 102, "max_ms": 98.685, "p95_ms": 88.025}, "visualize": {"avg_ms": 11.133, "count": 102, "max_ms": 13.315, "p95_ms": 12.185}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 412}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3999.852, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55db86c76a80] error while decoding MB 76 30, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:06:31+09:00", "last_error_summary": "[h264 @ 0x55db86c76a80] error while decoding MB 76 30, bytestream -7", "last_exit_at": "2026-07-01T18:04:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.951, "last_started_at": "2026-07-01T18:04:41+09:00", "pid": 14296, "process": {"exists": false, "pid": 14296}, "restart_count": 99, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 100}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.061, "sample_count": 1137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49502, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:06:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T18:06:31+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T18:06:31+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T18:06:31.036304+09:00", "mtime_epoch": 1782896791.0363045, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186044}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:06:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 879.668, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.154, "pushed": 4.154, "read": 4.154}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.027, "count": 124, "max_ms": 10.511, "p95_ms": 9.854}, "gstreamer_push": {"avg_ms": 0.944, "count": 124, "max_ms": 3.003, "p95_ms": 1.817}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.733, "count": 124, "max_ms": 69.442, "p95_ms": 61.181}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.315, "count": 124, "max_ms": 93.082, "p95_ms": 86.462}, "yolo11_640_fire_11class": {"avg_ms": 54.305, "count": 124, "max_ms": 73.88, "p95_ms": 71.512}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.041, "count": 124, "max_ms": 92.123, "p95_ms": 88.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.297, "count": 124, "max_ms": 329.819, "p95_ms": 262.249}, "read_interval": {"avg_ms": 242.197, "count": 124, "max_ms": 429.905, "p95_ms": 278.801}, "triton_wait": {"avg_ms": 68.988, "count": 124, "max_ms": 93.095, "p95_ms": 88.286}, "visualize": {"avg_ms": 10.752, "count": 124, "max_ms": 14.09, "p95_ms": 11.531}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.852, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 412}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3999.102, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:06:31+09:00", "last_error_summary": "[h264 @ 0x55db86c76a80] error while decoding MB 76 30, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:06:31+09:00", "last_error_summary": "[h264 @ 0x55db86c76a80] error while decoding MB 76 30, bytestream -7", "last_exit_at": "2026-07-01T18:06:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.125, "last_started_at": "2026-07-01T18:04:41+09:00", "pid": 14296, "process": {"exists": false, "pid": 14296}, "restart_count": 99, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 100}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.061, "sample_count": 1138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49503, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:06:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:06:31+09:00", "last_frame_read_age_sec": 0.226, "last_frame_read_at": "2026-07-01T18:06:31+09:00", "mjpeg": {"age_sec": 0.219, "exists": true, "mtime": "2026-07-01T18:06:31.036304+09:00", "mtime_epoch": 1782896791.0363045, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186044}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:06:31+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 879.668, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.162, "pushed": 4.162, "read": 4.129}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.025, "count": 124, "max_ms": 10.511, "p95_ms": 9.854}, "gstreamer_push": {"avg_ms": 0.944, "count": 124, "max_ms": 3.003, "p95_ms": 1.817}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.659, "count": 124, "max_ms": 69.442, "p95_ms": 61.181}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.243, "count": 124, "max_ms": 93.082, "p95_ms": 86.462}, "yolo11_640_fire_11class": {"avg_ms": 54.243, "count": 124, "max_ms": 73.88, "p95_ms": 71.512}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.979, "count": 124, "max_ms": 92.123, "p95_ms": 88.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.214, "count": 124, "max_ms": 329.819, "p95_ms": 262.249}, "read_interval": {"avg_ms": 242.198, "count": 123, "max_ms": 429.905, "p95_ms": 278.801}, "triton_wait": {"avg_ms": 68.916, "count": 124, "max_ms": 93.095, "p95_ms": 88.286}, "visualize": {"avg_ms": 10.752, "count": 124, "max_ms": 14.09, "p95_ms": 11.531}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.79, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 412}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3999.094, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:06:31+09:00", "last_error_summary": "[h264 @ 0x55db86c76a80] error while decoding MB 76 30, bytestream -7", "last_exit_at": "2026-07-01T18:06:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.128, "last_started_at": "2026-07-01T18:04:41+09:00", "pid": 14296, "process": {"exists": false, "pid": 14296}, "restart_count": 99, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 100}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.061, "sample_count": 1138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49503, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:06:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:06:31+09:00", "last_frame_read_age_sec": 0.229, "last_frame_read_at": "2026-07-01T18:06:31+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T18:06:31.036304+09:00", "mtime_epoch": 1782896791.0363045, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186044}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:06:31+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 879.668, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.162, "pushed": 4.162, "read": 4.128}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.025, "count": 124, "max_ms": 10.511, "p95_ms": 9.854}, "gstreamer_push": {"avg_ms": 0.944, "count": 124, "max_ms": 3.003, "p95_ms": 1.817}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.659, "count": 124, "max_ms": 69.442, "p95_ms": 61.181}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.243, "count": 124, "max_ms": 93.082, "p95_ms": 86.462}, "yolo11_640_fire_11class": {"avg_ms": 54.243, "count": 124, "max_ms": 73.88, "p95_ms": 71.512}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.979, "count": 124, "max_ms": 92.123, "p95_ms": 88.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.214, "count": 124, "max_ms": 329.819, "p95_ms": 262.249}, "read_interval": {"avg_ms": 242.198, "count": 123, "max_ms": 429.905, "p95_ms": 278.801}, "triton_wait": {"avg_ms": 68.916, "count": 124, "max_ms": 93.095, "p95_ms": 88.286}, "visualize": {"avg_ms": 10.752, "count": 124, "max_ms": 14.09, "p95_ms": 11.531}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.793, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 412}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4106.922, "limit_mb": 8796093022207.996}, "detail": {"pid": 14403, "restart_count": 100, "start_count": 101}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:06:31+09:00", "last_error_summary": "[h264 @ 0x55db86c76a80] error while decoding MB 76 30, bytestream -7", "last_exit_at": "2026-07-01T18:06:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:06:34+09:00", "pid": 14403, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.117, "name": "ffmpeg", "pid": 14403, "rss_mb": 136.117, "state": "S (sleeping)", "threads": 106}, "restart_count": 100, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 101}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.982, "sample_count": 1119, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49503, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:06:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.904, "last_frame_pushed_at": "2026-07-01T18:06:31+09:00", "last_frame_read_age_sec": 5.127, "last_frame_read_at": "2026-07-01T18:06:31+09:00", "mjpeg": {"age_sec": 4.887, "exists": true, "mtime": "2026-07-01T18:06:31.268308+09:00", "mtime_epoch": 1782896791.268308, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186155}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:06:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 879.668, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.474, "pushed": 3.474, "read": 3.441}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.049, "count": 103, "max_ms": 10.511, "p95_ms": 9.862}, "gstreamer_push": {"avg_ms": 0.945, "count": 104, "max_ms": 3.003, "p95_ms": 1.861}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.186, "count": 103, "max_ms": 69.442, "p95_ms": 61.181}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.842, "count": 103, "max_ms": 93.082, "p95_ms": 85.839}, "yolo11_640_fire_11class": {"avg_ms": 53.928, "count": 103, "max_ms": 73.88, "p95_ms": 71.265}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.808, "count": 103, "max_ms": 92.123, "p95_ms": 88.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.504, "count": 104, "max_ms": 329.819, "p95_ms": 262.249}, "read_interval": {"avg_ms": 242.807, "count": 103, "max_ms": 429.905, "p95_ms": 278.801}, "triton_wait": {"avg_ms": 68.631, "count": 103, "max_ms": 93.095, "p95_ms": 88.286}, "visualize": {"avg_ms": 10.821, "count": 103, "max_ms": 14.09, "p95_ms": 11.772}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 416}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4000.355, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:06:31+09:00", "last_error_summary": "[h264 @ 0x55db86c76a80] error while decoding MB 76 30, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:06:31+09:00", "last_error_summary": "[h264 @ 0x55db86c76a80] error while decoding MB 76 30, bytestream -7", "last_exit_at": "2026-07-01T18:08:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.941, "last_started_at": "2026-07-01T18:06:34+09:00", "pid": 14403, "process": {"exists": false, "pid": 14403}, "restart_count": 100, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 101}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 261.097, "sample_count": 1222, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49960, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:08:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:08:23+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T18:08:22+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T18:08:22.873967+09:00", "mtime_epoch": 1782896902.8739672, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186636}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:08:23+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 882.738, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.338, "pushed": 4.338, "read": 4.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.559, "count": 129, "max_ms": 10.788, "p95_ms": 10.125}, "gstreamer_push": {"avg_ms": 0.94, "count": 130, "max_ms": 2.913, "p95_ms": 1.776}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.266, "count": 129, "max_ms": 64.873, "p95_ms": 60.989}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.549, "count": 129, "max_ms": 91.656, "p95_ms": 87.159}, "yolo11_640_fire_11class": {"avg_ms": 55.662, "count": 129, "max_ms": 75.054, "p95_ms": 72.671}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.995, "count": 129, "max_ms": 92.442, "p95_ms": 87.072}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.596, "count": 130, "max_ms": 288.719, "p95_ms": 248.426}, "read_interval": {"avg_ms": 232.141, "count": 129, "max_ms": 301.302, "p95_ms": 259.54}, "triton_wait": {"avg_ms": 70.524, "count": 129, "max_ms": 92.452, "p95_ms": 88.329}, "visualize": {"avg_ms": 10.264, "count": 129, "max_ms": 13.536, "p95_ms": 11.359}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 416}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4000.348, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:06:31+09:00", "last_error_summary": "[h264 @ 0x55db86c76a80] error while decoding MB 76 30, bytestream -7", "last_exit_at": "2026-07-01T18:08:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.944, "last_started_at": "2026-07-01T18:06:34+09:00", "pid": 14403, "process": {"exists": false, "pid": 14403}, "restart_count": 100, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 101}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 261.097, "sample_count": 1222, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49960, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:08:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:08:23+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T18:08:22+09:00", "mjpeg": {"age_sec": 0.225, "exists": true, "mtime": "2026-07-01T18:08:22.873967+09:00", "mtime_epoch": 1782896902.8739672, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186636}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:08:23+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 882.738, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.338, "pushed": 4.338, "read": 4.304}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.559, "count": 129, "max_ms": 10.788, "p95_ms": 10.125}, "gstreamer_push": {"avg_ms": 0.94, "count": 130, "max_ms": 2.913, "p95_ms": 1.776}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.266, "count": 129, "max_ms": 64.873, "p95_ms": 60.989}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.549, "count": 129, "max_ms": 91.656, "p95_ms": 87.159}, "yolo11_640_fire_11class": {"avg_ms": 55.662, "count": 129, "max_ms": 75.054, "p95_ms": 72.671}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.995, "count": 129, "max_ms": 92.442, "p95_ms": 87.072}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.596, "count": 130, "max_ms": 288.719, "p95_ms": 248.426}, "read_interval": {"avg_ms": 232.141, "count": 129, "max_ms": 301.302, "p95_ms": 259.54}, "triton_wait": {"avg_ms": 70.524, "count": 129, "max_ms": 92.452, "p95_ms": 88.329}, "visualize": {"avg_ms": 10.264, "count": 129, "max_ms": 13.536, "p95_ms": 11.359}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 416}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4107.898, "limit_mb": 8796093022207.996}, "detail": {"pid": 14510, "restart_count": 101, "start_count": 102}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:06:31+09:00", "last_error_summary": "[h264 @ 0x55db86c76a80] error while decoding MB 76 30, bytestream -7", "last_exit_at": "2026-07-01T18:08:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:08:26+09:00", "pid": 14510, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.961, "name": "ffmpeg", "pid": 14510, "rss_mb": 136.961, "state": "S (sleeping)", "threads": 106}, "restart_count": 101, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 102}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 261.097, "sample_count": 1202, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 49960, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:08:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.953, "last_frame_pushed_at": "2026-07-01T18:08:23+09:00", "last_frame_read_age_sec": 5.181, "last_frame_read_at": "2026-07-01T18:08:22+09:00", "mjpeg": {"age_sec": 4.936, "exists": true, "mtime": "2026-07-01T18:08:23.109971+09:00", "mtime_epoch": 1782896903.1099708, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186726}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:08:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 882.738, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.605, "pushed": 3.605, "read": 3.571}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.656, "count": 107, "max_ms": 10.788, "p95_ms": 10.189}, "gstreamer_push": {"avg_ms": 0.939, "count": 108, "max_ms": 2.913, "p95_ms": 1.762}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.994, "count": 107, "max_ms": 64.873, "p95_ms": 60.709}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.068, "count": 107, "max_ms": 91.656, "p95_ms": 86.894}, "yolo11_640_fire_11class": {"avg_ms": 54.915, "count": 107, "max_ms": 74.275, "p95_ms": 70.948}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.422, "count": 107, "max_ms": 92.442, "p95_ms": 86.563}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.067, "count": 108, "max_ms": 288.719, "p95_ms": 248.426}, "read_interval": {"avg_ms": 233.711, "count": 107, "max_ms": 301.302, "p95_ms": 259.54}, "triton_wait": {"avg_ms": 69.92, "count": 107, "max_ms": 92.452, "p95_ms": 88.329}, "visualize": {"avg_ms": 10.366, "count": 107, "max_ms": 13.536, "p95_ms": 11.749}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 420}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4006.574, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ff42bb5f00] error while decoding MB 60 9, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:10:15+09:00", "last_error_summary": "[h264 @ 0x55ff42bb5f00] error while decoding MB 60 9, bytestream -5", "last_exit_at": "2026-07-01T18:08:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.366, "last_started_at": "2026-07-01T18:08:26+09:00", "pid": 14510, "process": {"exists": false, "pid": 14510}, "restart_count": 101, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 102}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.811, "sample_count": 1215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50400, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:10:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T18:10:15+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T18:10:15+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T18:10:15.367651+09:00", "mtime_epoch": 1782897015.3676512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188981}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:10:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 884.93, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.203, "pushed": 4.203, "read": 4.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.916, "count": 125, "max_ms": 12.125, "p95_ms": 9.99}, "gstreamer_push": {"avg_ms": 0.921, "count": 126, "max_ms": 2.844, "p95_ms": 1.743}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.312, "count": 125, "max_ms": 71.453, "p95_ms": 62.164}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.631, "count": 125, "max_ms": 90.383, "p95_ms": 86.424}, "yolo11_640_fire_11class": {"avg_ms": 55.293, "count": 125, "max_ms": 74.41, "p95_ms": 70.275}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.978, "count": 125, "max_ms": 92.883, "p95_ms": 87.824}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.919, "count": 126, "max_ms": 292.843, "p95_ms": 253.784}, "read_interval": {"avg_ms": 239.342, "count": 126, "max_ms": 302.827, "p95_ms": 262.84}, "triton_wait": {"avg_ms": 69.651, "count": 125, "max_ms": 92.895, "p95_ms": 88.01}, "visualize": {"avg_ms": 10.722, "count": 125, "max_ms": 13.229, "p95_ms": 11.466}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 420}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4005.516, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:10:15+09:00", "last_error_summary": "[h264 @ 0x55ff42bb5f00] error while decoding MB 60 9, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:10:15+09:00", "last_error_summary": "[h264 @ 0x55ff42bb5f00] error while decoding MB 60 9, bytestream -5", "last_exit_at": "2026-07-01T18:10:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.537, "last_started_at": "2026-07-01T18:08:26+09:00", "pid": 14510, "process": {"exists": false, "pid": 14510}, "restart_count": 101, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 102}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.811, "sample_count": 1215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50401, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:10:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:10:15+09:00", "last_frame_read_age_sec": 0.223, "last_frame_read_at": "2026-07-01T18:10:15+09:00", "mjpeg": {"age_sec": 0.214, "exists": true, "mtime": "2026-07-01T18:10:15.367651+09:00", "mtime_epoch": 1782897015.3676512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188981}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:10:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 884.93, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.211, "pushed": 4.211, "read": 4.178}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.908, "count": 125, "max_ms": 12.125, "p95_ms": 9.99}, "gstreamer_push": {"avg_ms": 0.921, "count": 126, "max_ms": 2.844, "p95_ms": 1.743}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.251, "count": 125, "max_ms": 71.453, "p95_ms": 62.164}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.56, "count": 125, "max_ms": 90.383, "p95_ms": 86.424}, "yolo11_640_fire_11class": {"avg_ms": 55.23, "count": 125, "max_ms": 74.41, "p95_ms": 70.275}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.912, "count": 125, "max_ms": 92.883, "p95_ms": 87.824}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.851, "count": 126, "max_ms": 292.843, "p95_ms": 253.784}, "read_interval": {"avg_ms": 239.368, "count": 125, "max_ms": 302.827, "p95_ms": 262.84}, "triton_wait": {"avg_ms": 69.585, "count": 125, "max_ms": 92.895, "p95_ms": 88.01}, "visualize": {"avg_ms": 10.733, "count": 125, "max_ms": 13.229, "p95_ms": 11.588}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 420}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4005.633, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:10:15+09:00", "last_error_summary": "[h264 @ 0x55ff42bb5f00] error while decoding MB 60 9, bytestream -5", "last_exit_at": "2026-07-01T18:10:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.54, "last_started_at": "2026-07-01T18:08:26+09:00", "pid": 14510, "process": {"exists": false, "pid": 14510}, "restart_count": 101, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 102}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.811, "sample_count": 1215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50401, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:10:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:10:15+09:00", "last_frame_read_age_sec": 0.226, "last_frame_read_at": "2026-07-01T18:10:15+09:00", "mjpeg": {"age_sec": 0.217, "exists": true, "mtime": "2026-07-01T18:10:15.367651+09:00", "mtime_epoch": 1782897015.3676512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188981}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:10:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 884.93, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.211, "pushed": 4.211, "read": 4.178}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.908, "count": 125, "max_ms": 12.125, "p95_ms": 9.99}, "gstreamer_push": {"avg_ms": 0.921, "count": 126, "max_ms": 2.844, "p95_ms": 1.743}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.251, "count": 125, "max_ms": 71.453, "p95_ms": 62.164}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.56, "count": 125, "max_ms": 90.383, "p95_ms": 86.424}, "yolo11_640_fire_11class": {"avg_ms": 55.23, "count": 125, "max_ms": 74.41, "p95_ms": 70.275}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.912, "count": 125, "max_ms": 92.883, "p95_ms": 87.824}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.851, "count": 126, "max_ms": 292.843, "p95_ms": 253.784}, "read_interval": {"avg_ms": 239.368, "count": 125, "max_ms": 302.827, "p95_ms": 262.84}, "triton_wait": {"avg_ms": 69.585, "count": 125, "max_ms": 92.895, "p95_ms": 88.01}, "visualize": {"avg_ms": 10.733, "count": 125, "max_ms": 13.229, "p95_ms": 11.588}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 420}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4111.398, "limit_mb": 8796093022207.996}, "detail": {"pid": 14617, "restart_count": 102, "start_count": 103}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:10:15+09:00", "last_error_summary": "[h264 @ 0x55ff42bb5f00] error while decoding MB 60 9, bytestream -5", "last_exit_at": "2026-07-01T18:10:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:10:18+09:00", "pid": 14617, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.629, "name": "ffmpeg", "pid": 14617, "rss_mb": 136.629, "state": "S (sleeping)", "threads": 106}, "restart_count": 102, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 103}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 264.665, "sample_count": 1198, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50401, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:10:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.459, "last_frame_pushed_at": "2026-07-01T18:10:15+09:00", "last_frame_read_age_sec": 4.678, "last_frame_read_at": "2026-07-01T18:10:15+09:00", "mjpeg": {"age_sec": 4.442, "exists": true, "mtime": "2026-07-01T18:10:15.595655+09:00", "mtime_epoch": 1782897015.5956547, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188850}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:10:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 884.93, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.589, "pushed": 3.589, "read": 3.556}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.91, "count": 107, "max_ms": 10.149, "p95_ms": 9.817}, "gstreamer_push": {"avg_ms": 0.923, "count": 107, "max_ms": 2.844, "p95_ms": 1.743}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.85, "count": 107, "max_ms": 71.453, "p95_ms": 62.164}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.552, "count": 107, "max_ms": 90.383, "p95_ms": 87.44}, "yolo11_640_fire_11class": {"avg_ms": 55.253, "count": 107, "max_ms": 74.41, "p95_ms": 71.023}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.945, "count": 107, "max_ms": 92.883, "p95_ms": 87.847}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.788, "count": 107, "max_ms": 292.843, "p95_ms": 258.579}, "read_interval": {"avg_ms": 239.31, "count": 106, "max_ms": 302.827, "p95_ms": 268.986}, "triton_wait": {"avg_ms": 69.632, "count": 107, "max_ms": 92.895, "p95_ms": 88.065}, "visualize": {"avg_ms": 10.713, "count": 107, "max_ms": 13.229, "p95_ms": 11.67}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.811, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 423}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4009.98, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ddf6bc2f80] error while decoding MB 49 36, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 90, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:12:07+09:00", "last_error_summary": "[h264 @ 0x55ddf6bc2f80] error while decoding MB 49 36, bytestream -7", "last_exit_at": "2026-07-01T18:10:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.496, "last_started_at": "2026-07-01T18:10:18+09:00", "pid": 14617, "process": {"exists": false, "pid": 14617}, "restart_count": 102, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 103}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.194, "sample_count": 1205, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50841, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:12:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T18:12:07+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T18:12:07+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T18:12:07.489339+09:00", "mtime_epoch": 1782897127.489339, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188843}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:12:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.844, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.087, "pushed": 4.087, "read": 4.087}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.353, "count": 121, "max_ms": 14.169, "p95_ms": 10.75}, "gstreamer_push": {"avg_ms": 0.964, "count": 122, "max_ms": 3.065, "p95_ms": 1.832}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.457, "count": 121, "max_ms": 69.897, "p95_ms": 60.746}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.189, "count": 121, "max_ms": 96.21, "p95_ms": 87.249}, "yolo11_640_fire_11class": {"avg_ms": 53.226, "count": 121, "max_ms": 78.75, "p95_ms": 68.628}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.586, "count": 121, "max_ms": 95.135, "p95_ms": 88.458}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.463, "count": 122, "max_ms": 336.854, "p95_ms": 261.327}, "read_interval": {"avg_ms": 246.229, "count": 122, "max_ms": 349.835, "p95_ms": 273.474}, "triton_wait": {"avg_ms": 68.036, "count": 121, "max_ms": 96.22, "p95_ms": 89.289}, "visualize": {"avg_ms": 11.219, "count": 121, "max_ms": 16.631, "p95_ms": 13.377}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.848, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 423}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4009.055, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:12:07+09:00", "last_error_summary": "[h264 @ 0x55ddf6bc2f80] error while decoding MB 49 36, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 90, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:12:07+09:00", "last_error_summary": "[h264 @ 0x55ddf6bc2f80] error while decoding MB 49 36, bytestream -7", "last_exit_at": "2026-07-01T18:12:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.694, "last_started_at": "2026-07-01T18:10:18+09:00", "pid": 14617, "process": {"exists": false, "pid": 14617}, "restart_count": 102, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 103}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.194, "sample_count": 1205, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50842, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:12:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:12:07+09:00", "last_frame_read_age_sec": 0.249, "last_frame_read_at": "2026-07-01T18:12:07+09:00", "mjpeg": {"age_sec": 0.241, "exists": true, "mtime": "2026-07-01T18:12:07.489339+09:00", "mtime_epoch": 1782897127.489339, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188843}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:12:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.844, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.094, "pushed": 4.094, "read": 4.061}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.352, "count": 122, "max_ms": 14.169, "p95_ms": 10.75}, "gstreamer_push": {"avg_ms": 0.964, "count": 122, "max_ms": 3.065, "p95_ms": 1.832}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.592, "count": 122, "max_ms": 69.897, "p95_ms": 61.252}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.341, "count": 122, "max_ms": 96.21, "p95_ms": 87.249}, "yolo11_640_fire_11class": {"avg_ms": 53.396, "count": 122, "max_ms": 78.75, "p95_ms": 69.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.729, "count": 122, "max_ms": 95.135, "p95_ms": 88.458}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.473, "count": 122, "max_ms": 336.854, "p95_ms": 261.327}, "read_interval": {"avg_ms": 246.164, "count": 121, "max_ms": 349.835, "p95_ms": 273.474}, "triton_wait": {"avg_ms": 68.182, "count": 122, "max_ms": 96.22, "p95_ms": 89.289}, "visualize": {"avg_ms": 11.214, "count": 122, "max_ms": 16.631, "p95_ms": 13.377}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.798, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 423}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4009.023, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 90, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:12:07+09:00", "last_error_summary": "[h264 @ 0x55ddf6bc2f80] error while decoding MB 49 36, bytestream -7", "last_exit_at": "2026-07-01T18:12:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.697, "last_started_at": "2026-07-01T18:10:18+09:00", "pid": 14617, "process": {"exists": false, "pid": 14617}, "restart_count": 102, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 103}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.194, "sample_count": 1205, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50842, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:12:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:12:07+09:00", "last_frame_read_age_sec": 0.252, "last_frame_read_at": "2026-07-01T18:12:07+09:00", "mjpeg": {"age_sec": 0.244, "exists": true, "mtime": "2026-07-01T18:12:07.489339+09:00", "mtime_epoch": 1782897127.489339, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188843}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:12:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.844, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.094, "pushed": 4.094, "read": 4.06}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.352, "count": 122, "max_ms": 14.169, "p95_ms": 10.75}, "gstreamer_push": {"avg_ms": 0.964, "count": 122, "max_ms": 3.065, "p95_ms": 1.832}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.592, "count": 122, "max_ms": 69.897, "p95_ms": 61.252}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.341, "count": 122, "max_ms": 96.21, "p95_ms": 87.249}, "yolo11_640_fire_11class": {"avg_ms": 53.396, "count": 122, "max_ms": 78.75, "p95_ms": 69.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.729, "count": 122, "max_ms": 95.135, "p95_ms": 88.458}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.473, "count": 122, "max_ms": 336.854, "p95_ms": 261.327}, "read_interval": {"avg_ms": 246.164, "count": 121, "max_ms": 349.835, "p95_ms": 273.474}, "triton_wait": {"avg_ms": 68.182, "count": 122, "max_ms": 96.22, "p95_ms": 89.289}, "visualize": {"avg_ms": 11.214, "count": 122, "max_ms": 16.631, "p95_ms": 13.377}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.801, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 423}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4114.902, "limit_mb": 8796093022207.996}, "detail": {"pid": 14724, "restart_count": 103, "start_count": 104}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 90, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:12:07+09:00", "last_error_summary": "[h264 @ 0x55ddf6bc2f80] error while decoding MB 49 36, bytestream -7", "last_exit_at": "2026-07-01T18:12:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:12:10+09:00", "pid": 14724, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.773, "name": "ffmpeg", "pid": 14724, "rss_mb": 136.773, "state": "S (sleeping)", "threads": 106}, "restart_count": 103, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 104}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 267.811, "sample_count": 1187, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 50842, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:12:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.291, "last_frame_pushed_at": "2026-07-01T18:12:07+09:00", "last_frame_read_age_sec": 4.538, "last_frame_read_at": "2026-07-01T18:12:07+09:00", "mjpeg": {"age_sec": 4.273, "exists": true, "mtime": "2026-07-01T18:12:07.745343+09:00", "mtime_epoch": 1782897127.745343, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188919}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:12:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.844, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.507, "pushed": 3.507, "read": 3.473}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.226, "count": 104, "max_ms": 12.801, "p95_ms": 10.49}, "gstreamer_push": {"avg_ms": 0.97, "count": 105, "max_ms": 3.065, "p95_ms": 1.832}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.98, "count": 104, "max_ms": 69.897, "p95_ms": 61.628}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.472, "count": 104, "max_ms": 96.21, "p95_ms": 89.277}, "yolo11_640_fire_11class": {"avg_ms": 53.498, "count": 104, "max_ms": 78.75, "p95_ms": 68.628}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.861, "count": 104, "max_ms": 95.135, "p95_ms": 88.458}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.712, "count": 105, "max_ms": 336.854, "p95_ms": 264.145}, "read_interval": {"avg_ms": 246.432, "count": 104, "max_ms": 349.835, "p95_ms": 273.823}, "triton_wait": {"avg_ms": 68.301, "count": 104, "max_ms": 96.22, "p95_ms": 89.289}, "visualize": {"avg_ms": 11.195, "count": 104, "max_ms": 16.631, "p95_ms": 13.377}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 427}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4003.266, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5593e03aaac0] error while decoding MB 53 6, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:14:00+09:00", "last_error_summary": "[h264 @ 0x5593e03aaac0] error while decoding MB 53 6, bytestream -6", "last_exit_at": "2026-07-01T18:12:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.888, "last_started_at": "2026-07-01T18:12:10+09:00", "pid": 14724, "process": {"exists": false, "pid": 14724}, "restart_count": 103, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 104}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.469, "sample_count": 1188, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51278, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:14:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T18:14:00+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T18:14:00+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T18:14:00.863053+09:00", "mtime_epoch": 1782897240.8630533, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192132}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:14:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 882.805, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.977, "pushed": 3.977, "read": 3.977}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.679, "count": 118, "max_ms": 12.488, "p95_ms": 10.424}, "gstreamer_push": {"avg_ms": 0.92, "count": 119, "max_ms": 2.874, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.991, "count": 118, "max_ms": 63.035, "p95_ms": 60.126}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.814, "count": 118, "max_ms": 93.29, "p95_ms": 89.04}, "yolo11_640_fire_11class": {"avg_ms": 52.687, "count": 118, "max_ms": 72.86, "p95_ms": 68.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.397, "count": 118, "max_ms": 91.839, "p95_ms": 88.335}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.366, "count": 119, "max_ms": 339.584, "p95_ms": 278.119}, "read_interval": {"avg_ms": 252.991, "count": 119, "max_ms": 350.865, "p95_ms": 287.389}, "triton_wait": {"avg_ms": 68.38, "count": 118, "max_ms": 93.302, "p95_ms": 89.053}, "visualize": {"avg_ms": 11.361, "count": 118, "max_ms": 13.095, "p95_ms": 12.162}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 427}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4002.125, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:14:00+09:00", "last_error_summary": "[h264 @ 0x5593e03aaac0] error while decoding MB 53 6, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:14:00+09:00", "last_error_summary": "[h264 @ 0x5593e03aaac0] error while decoding MB 53 6, bytestream -6", "last_exit_at": "2026-07-01T18:14:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.066, "last_started_at": "2026-07-01T18:12:10+09:00", "pid": 14724, "process": {"exists": false, "pid": 14724}, "restart_count": 103, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 104}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.469, "sample_count": 1189, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51279, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:14:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:14:01+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T18:14:00+09:00", "mjpeg": {"age_sec": 0.221, "exists": true, "mtime": "2026-07-01T18:14:00.863053+09:00", "mtime_epoch": 1782897240.8630533, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192132}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:14:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 882.805, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.99, "pushed": 3.99, "read": 3.957}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.673, "count": 119, "max_ms": 12.488, "p95_ms": 10.424}, "gstreamer_push": {"avg_ms": 0.92, "count": 119, "max_ms": 2.874, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.93, "count": 119, "max_ms": 63.035, "p95_ms": 60.126}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.753, "count": 119, "max_ms": 93.29, "p95_ms": 89.04}, "yolo11_640_fire_11class": {"avg_ms": 52.623, "count": 119, "max_ms": 72.86, "p95_ms": 68.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.333, "count": 119, "max_ms": 91.839, "p95_ms": 88.335}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.263, "count": 119, "max_ms": 339.584, "p95_ms": 278.119}, "read_interval": {"avg_ms": 253.032, "count": 118, "max_ms": 350.865, "p95_ms": 287.389}, "triton_wait": {"avg_ms": 68.314, "count": 119, "max_ms": 93.302, "p95_ms": 89.053}, "visualize": {"avg_ms": 11.358, "count": 119, "max_ms": 13.095, "p95_ms": 12.162}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.821, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 427}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4002.242, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:14:00+09:00", "last_error_summary": "[h264 @ 0x5593e03aaac0] error while decoding MB 53 6, bytestream -6", "last_exit_at": "2026-07-01T18:14:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.07, "last_started_at": "2026-07-01T18:12:10+09:00", "pid": 14724, "process": {"exists": false, "pid": 14724}, "restart_count": 103, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 104}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.469, "sample_count": 1189, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51279, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:14:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:14:01+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-01T18:14:00+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T18:14:00.863053+09:00", "mtime_epoch": 1782897240.8630533, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192132}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:14:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 882.805, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.99, "pushed": 3.99, "read": 3.957}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.673, "count": 119, "max_ms": 12.488, "p95_ms": 10.424}, "gstreamer_push": {"avg_ms": 0.92, "count": 119, "max_ms": 2.874, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.93, "count": 119, "max_ms": 63.035, "p95_ms": 60.126}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.753, "count": 119, "max_ms": 93.29, "p95_ms": 89.04}, "yolo11_640_fire_11class": {"avg_ms": 52.623, "count": 119, "max_ms": 72.86, "p95_ms": 68.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.333, "count": 119, "max_ms": 91.839, "p95_ms": 88.335}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.263, "count": 119, "max_ms": 339.584, "p95_ms": 278.119}, "read_interval": {"avg_ms": 253.032, "count": 118, "max_ms": 350.865, "p95_ms": 287.389}, "triton_wait": {"avg_ms": 68.314, "count": 119, "max_ms": 93.302, "p95_ms": 89.053}, "visualize": {"avg_ms": 11.358, "count": 119, "max_ms": 13.095, "p95_ms": 12.162}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.824, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 427}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4121.215, "limit_mb": 8796093022207.996}, "detail": {"pid": 14831, "restart_count": 104, "start_count": 105}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:14:00+09:00", "last_error_summary": "[h264 @ 0x5593e03aaac0] error while decoding MB 53 6, bytestream -6", "last_exit_at": "2026-07-01T18:14:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:14:03+09:00", "pid": 14831, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.328, "name": "ffmpeg", "pid": 14831, "rss_mb": 147.328, "state": "S (sleeping)", "threads": 106}, "restart_count": 104, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 105}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.588, "sample_count": 1168, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51279, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:14:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.695, "last_frame_pushed_at": "2026-07-01T18:14:01+09:00", "last_frame_read_age_sec": 4.922, "last_frame_read_at": "2026-07-01T18:14:00+09:00", "mjpeg": {"age_sec": 4.677, "exists": true, "mtime": "2026-07-01T18:14:01.099057+09:00", "mtime_epoch": 1782897241.0990567, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185060}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:14:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 882.805, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.369, "pushed": 3.369, "read": 3.336}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.633, "count": 100, "max_ms": 12.488, "p95_ms": 10.242}, "gstreamer_push": {"avg_ms": 0.913, "count": 101, "max_ms": 2.871, "p95_ms": 0.923}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.019, "count": 100, "max_ms": 63.035, "p95_ms": 60.126}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.043, "count": 100, "max_ms": 93.29, "p95_ms": 88.337}, "yolo11_640_fire_11class": {"avg_ms": 52.812, "count": 100, "max_ms": 72.86, "p95_ms": 68.893}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.505, "count": 100, "max_ms": 91.839, "p95_ms": 88.335}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.145, "count": 101, "max_ms": 339.584, "p95_ms": 278.119}, "read_interval": {"avg_ms": 252.951, "count": 100, "max_ms": 350.865, "p95_ms": 287.389}, "triton_wait": {"avg_ms": 68.572, "count": 100, "max_ms": 93.302, "p95_ms": 88.947}, "visualize": {"avg_ms": 11.311, "count": 100, "max_ms": 13.095, "p95_ms": 12.161}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 431}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4008.855, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56239ed9f2c0] error while decoding MB 63 34, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:15:53+09:00", "last_error_summary": "[h264 @ 0x56239ed9f2c0] error while decoding MB 63 34, bytestream -9", "last_exit_at": "2026-07-01T18:14:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.122, "last_started_at": "2026-07-01T18:14:03+09:00", "pid": 14831, "process": {"exists": false, "pid": 14831}, "restart_count": 104, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 105}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.727, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51726, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:15:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T18:15:53+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T18:15:53+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T18:15:53.856768+09:00", "mtime_epoch": 1782897353.8567677, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183488}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:15:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.867, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.139, "pushed": 4.139, "read": 4.139}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.283, "count": 123, "max_ms": 15.762, "p95_ms": 10.084}, "gstreamer_push": {"avg_ms": 0.907, "count": 124, "max_ms": 2.457, "p95_ms": 0.96}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.321, "count": 123, "max_ms": 65.941, "p95_ms": 60.544}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.962, "count": 123, "max_ms": 90.09, "p95_ms": 83.854}, "yolo11_640_fire_11class": {"avg_ms": 52.936, "count": 123, "max_ms": 72.123, "p95_ms": 66.69}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.495, "count": 123, "max_ms": 90.914, "p95_ms": 84.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.418, "count": 124, "max_ms": 324.466, "p95_ms": 260.812}, "read_interval": {"avg_ms": 242.957, "count": 124, "max_ms": 334.385, "p95_ms": 270.883}, "triton_wait": {"avg_ms": 67.582, "count": 123, "max_ms": 90.928, "p95_ms": 87.65}, "visualize": {"avg_ms": 11.022, "count": 123, "max_ms": 15.66, "p95_ms": 12.234}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 431}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4008.324, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:15:53+09:00", "last_error_summary": "[h264 @ 0x56239ed9f2c0] error while decoding MB 63 34, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:15:53+09:00", "last_error_summary": "[h264 @ 0x56239ed9f2c0] error while decoding MB 63 34, bytestream -9", "last_exit_at": "2026-07-01T18:15:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.301, "last_started_at": "2026-07-01T18:14:03+09:00", "pid": 14831, "process": {"exists": false, "pid": 14831}, "restart_count": 104, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 105}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.727, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51727, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:15:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:15:54+09:00", "last_frame_read_age_sec": 0.229, "last_frame_read_at": "2026-07-01T18:15:53+09:00", "mjpeg": {"age_sec": 0.219, "exists": true, "mtime": "2026-07-01T18:15:53.856768+09:00", "mtime_epoch": 1782897353.8567677, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183488}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:15:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.867, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.147, "pushed": 4.147, "read": 4.113}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.288, "count": 123, "max_ms": 15.762, "p95_ms": 10.084}, "gstreamer_push": {"avg_ms": 0.907, "count": 124, "max_ms": 2.457, "p95_ms": 0.96}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.378, "count": 123, "max_ms": 65.941, "p95_ms": 60.544}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.972, "count": 123, "max_ms": 90.09, "p95_ms": 83.854}, "yolo11_640_fire_11class": {"avg_ms": 52.925, "count": 123, "max_ms": 72.123, "p95_ms": 66.69}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.496, "count": 123, "max_ms": 90.914, "p95_ms": 84.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.457, "count": 124, "max_ms": 324.466, "p95_ms": 260.812}, "read_interval": {"avg_ms": 243.069, "count": 123, "max_ms": 334.385, "p95_ms": 270.883}, "triton_wait": {"avg_ms": 67.584, "count": 123, "max_ms": 90.928, "p95_ms": 87.65}, "visualize": {"avg_ms": 11.011, "count": 123, "max_ms": 15.66, "p95_ms": 12.234}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 431}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4008.316, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:15:53+09:00", "last_error_summary": "[h264 @ 0x56239ed9f2c0] error while decoding MB 63 34, bytestream -9", "last_exit_at": "2026-07-01T18:15:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.304, "last_started_at": "2026-07-01T18:14:03+09:00", "pid": 14831, "process": {"exists": false, "pid": 14831}, "restart_count": 104, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 105}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.727, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51727, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:15:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:15:54+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-01T18:15:53+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T18:15:53.856768+09:00", "mtime_epoch": 1782897353.8567677, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183488}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:15:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.867, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.146, "pushed": 4.146, "read": 4.113}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.288, "count": 123, "max_ms": 15.762, "p95_ms": 10.084}, "gstreamer_push": {"avg_ms": 0.907, "count": 124, "max_ms": 2.457, "p95_ms": 0.96}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.378, "count": 123, "max_ms": 65.941, "p95_ms": 60.544}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.972, "count": 123, "max_ms": 90.09, "p95_ms": 83.854}, "yolo11_640_fire_11class": {"avg_ms": 52.925, "count": 123, "max_ms": 72.123, "p95_ms": 66.69}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.496, "count": 123, "max_ms": 90.914, "p95_ms": 84.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.457, "count": 124, "max_ms": 324.466, "p95_ms": 260.812}, "read_interval": {"avg_ms": 243.069, "count": 123, "max_ms": 334.385, "p95_ms": 270.883}, "triton_wait": {"avg_ms": 67.584, "count": 123, "max_ms": 90.928, "p95_ms": 87.65}, "visualize": {"avg_ms": 11.011, "count": 123, "max_ms": 15.66, "p95_ms": 12.234}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 431}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4114.273, "limit_mb": 8796093022207.996}, "detail": {"pid": 14938, "restart_count": 105, "start_count": 106}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 90, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:15:53+09:00", "last_error_summary": "[h264 @ 0x56239ed9f2c0] error while decoding MB 63 34, bytestream -9", "last_exit_at": "2026-07-01T18:15:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:15:57+09:00", "pid": 14938, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.484, "name": "ffmpeg", "pid": 14938, "rss_mb": 136.484, "state": "S (sleeping)", "threads": 106}, "restart_count": 105, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 106}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.727, "sample_count": 1164, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 51727, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:15:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.937, "last_frame_pushed_at": "2026-07-01T18:15:54+09:00", "last_frame_read_age_sec": 5.164, "last_frame_read_at": "2026-07-01T18:15:53+09:00", "mjpeg": {"age_sec": 4.921, "exists": true, "mtime": "2026-07-01T18:15:54.088771+09:00", "mtime_epoch": 1782897354.0887713, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183524}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:15:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.867, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.494, "pushed": 3.494, "read": 3.46}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.237, "count": 104, "max_ms": 13.942, "p95_ms": 9.962}, "gstreamer_push": {"avg_ms": 0.908, "count": 104, "max_ms": 2.457, "p95_ms": 0.96}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.598, "count": 104, "max_ms": 65.941, "p95_ms": 60.938}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.412, "count": 104, "max_ms": 90.09, "p95_ms": 85.283}, "yolo11_640_fire_11class": {"avg_ms": 53.136, "count": 104, "max_ms": 72.123, "p95_ms": 66.69}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.646, "count": 104, "max_ms": 88.042, "p95_ms": 84.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.598, "count": 104, "max_ms": 313.365, "p95_ms": 254.111}, "read_interval": {"avg_ms": 241.17, "count": 103, "max_ms": 322.599, "p95_ms": 263.208}, "triton_wait": {"avg_ms": 67.874, "count": 104, "max_ms": 90.103, "p95_ms": 87.65}, "visualize": {"avg_ms": 10.938, "count": 104, "max_ms": 15.66, "p95_ms": 11.883}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.765, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4004.551, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55dd28407e00] error while decoding MB 27 7, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 93, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:17:45+09:00", "last_error_summary": "[h264 @ 0x55dd28407e00] error while decoding MB 27 7, bytestream -7", "last_exit_at": "2026-07-01T18:15:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.929, "last_started_at": "2026-07-01T18:15:57+09:00", "pid": 14938, "process": {"exists": false, "pid": 14938}, "restart_count": 105, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 106}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.288, "sample_count": 1184, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 52161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:17:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T18:17:45+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T18:17:45+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T18:17:45.898473+09:00", "mtime_epoch": 1782897465.8984725, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188368}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:17:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.609, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.187, "pushed": 4.187, "read": 4.187}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.967, "count": 125, "max_ms": 14.277, "p95_ms": 9.958}, "gstreamer_push": {"avg_ms": 0.898, "count": 125, "max_ms": 1.891, "p95_ms": 0.939}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.592, "count": 125, "max_ms": 68.626, "p95_ms": 63.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.231, "count": 125, "max_ms": 92.598, "p95_ms": 89.076}, "yolo11_640_fire_11class": {"avg_ms": 54.678, "count": 125, "max_ms": 75.892, "p95_ms": 69.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.358, "count": 125, "max_ms": 91.029, "p95_ms": 89.385}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.618, "count": 125, "max_ms": 309.098, "p95_ms": 254.29}, "read_interval": {"avg_ms": 240.04, "count": 125, "max_ms": 319.21, "p95_ms": 264.264}, "triton_wait": {"avg_ms": 70.087, "count": 125, "max_ms": 92.61, "p95_ms": 90.398}, "visualize": {"avg_ms": 10.739, "count": 125, "max_ms": 13.603, "p95_ms": 11.603}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.852, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4003.73, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:17:45+09:00", "last_error_summary": "[h264 @ 0x55dd28407e00] error while decoding MB 27 7, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 93, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:17:45+09:00", "last_error_summary": "[h264 @ 0x55dd28407e00] error while decoding MB 27 7, bytestream -7", "last_exit_at": "2026-07-01T18:17:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.086, "last_started_at": "2026-07-01T18:15:57+09:00", "pid": 14938, "process": {"exists": false, "pid": 14938}, "restart_count": 105, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 106}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.288, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 52162, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:17:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:17:46+09:00", "last_frame_read_age_sec": 0.208, "last_frame_read_at": "2026-07-01T18:17:45+09:00", "mjpeg": {"age_sec": 0.196, "exists": true, "mtime": "2026-07-01T18:17:45.898473+09:00", "mtime_epoch": 1782897465.8984725, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188368}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:17:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.609, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.167, "pushed": 4.167, "read": 4.167}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.967, "count": 125, "max_ms": 14.277, "p95_ms": 9.958}, "gstreamer_push": {"avg_ms": 0.898, "count": 125, "max_ms": 1.891, "p95_ms": 0.939}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.631, "count": 125, "max_ms": 68.626, "p95_ms": 63.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.234, "count": 125, "max_ms": 92.598, "p95_ms": 89.076}, "yolo11_640_fire_11class": {"avg_ms": 54.663, "count": 125, "max_ms": 75.892, "p95_ms": 69.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.338, "count": 125, "max_ms": 91.029, "p95_ms": 89.385}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.616, "count": 125, "max_ms": 309.098, "p95_ms": 254.29}, "read_interval": {"avg_ms": 240.04, "count": 125, "max_ms": 319.21, "p95_ms": 264.264}, "triton_wait": {"avg_ms": 70.08, "count": 125, "max_ms": 92.61, "p95_ms": 90.398}, "visualize": {"avg_ms": 10.735, "count": 125, "max_ms": 13.603, "p95_ms": 11.603}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4003.805, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 93, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:17:45+09:00", "last_error_summary": "[h264 @ 0x55dd28407e00] error while decoding MB 27 7, bytestream -7", "last_exit_at": "2026-07-01T18:17:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.089, "last_started_at": "2026-07-01T18:15:57+09:00", "pid": 14938, "process": {"exists": false, "pid": 14938}, "restart_count": 105, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 106}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.288, "sample_count": 1185, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 52162, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:17:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:17:46+09:00", "last_frame_read_age_sec": 0.212, "last_frame_read_at": "2026-07-01T18:17:45+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T18:17:45.898473+09:00", "mtime_epoch": 1782897465.8984725, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188368}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:17:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.609, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.197, "pushed": 4.197, "read": 4.163}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.967, "count": 125, "max_ms": 14.277, "p95_ms": 9.958}, "gstreamer_push": {"avg_ms": 0.898, "count": 125, "max_ms": 1.891, "p95_ms": 0.939}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.631, "count": 125, "max_ms": 68.626, "p95_ms": 63.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.234, "count": 125, "max_ms": 92.598, "p95_ms": 89.076}, "yolo11_640_fire_11class": {"avg_ms": 54.663, "count": 125, "max_ms": 75.892, "p95_ms": 69.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.338, "count": 125, "max_ms": 91.029, "p95_ms": 89.385}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.616, "count": 125, "max_ms": 309.098, "p95_ms": 254.29}, "read_interval": {"avg_ms": 240.243, "count": 124, "max_ms": 319.21, "p95_ms": 264.264}, "triton_wait": {"avg_ms": 70.08, "count": 125, "max_ms": 92.61, "p95_ms": 90.398}, "visualize": {"avg_ms": 10.735, "count": 125, "max_ms": 13.603, "p95_ms": 11.603}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.784, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 434}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4117.109, "limit_mb": 8796093022207.996}, "detail": {"pid": 15045, "restart_count": 106, "start_count": 107}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 91, "h264_decode_error_count": 93, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:17:45+09:00", "last_error_summary": "[h264 @ 0x55dd28407e00] error while decoding MB 27 7, bytestream -7", "last_exit_at": "2026-07-01T18:17:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:17:49+09:00", "pid": 15045, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.141, "name": "ffmpeg", "pid": 15045, "rss_mb": 141.141, "state": "S (sleeping)", "threads": 106}, "restart_count": 106, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 107}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.573, "sample_count": 1165, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 52162, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:17:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 5.007, "last_frame_pushed_at": "2026-07-01T18:17:46+09:00", "last_frame_read_age_sec": 5.212, "last_frame_read_at": "2026-07-01T18:17:45+09:00", "mjpeg": {"age_sec": 4.984, "exists": true, "mtime": "2026-07-01T18:17:46.114476+09:00", "mtime_epoch": 1782897466.1144757, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188474}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:17:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.609, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.469, "pushed": 3.469, "read": 3.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.975, "count": 103, "max_ms": 14.277, "p95_ms": 9.958}, "gstreamer_push": {"avg_ms": 0.903, "count": 104, "max_ms": 1.891, "p95_ms": 0.939}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.029, "count": 103, "max_ms": 68.626, "p95_ms": 63.491}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.499, "count": 103, "max_ms": 92.598, "p95_ms": 89.076}, "yolo11_640_fire_11class": {"avg_ms": 54.872, "count": 103, "max_ms": 75.892, "p95_ms": 69.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.57, "count": 103, "max_ms": 91.029, "p95_ms": 89.385}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.75, "count": 104, "max_ms": 309.098, "p95_ms": 254.29}, "read_interval": {"avg_ms": 242.653, "count": 103, "max_ms": 319.21, "p95_ms": 264.264}, "triton_wait": {"avg_ms": 70.333, "count": 103, "max_ms": 92.61, "p95_ms": 90.398}, "visualize": {"avg_ms": 10.857, "count": 103, "max_ms": 13.603, "p95_ms": 11.603}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 438}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4007.375, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d52d2667c0] error while decoding MB 37 34, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:19:45+09:00", "last_error_summary": "[h264 @ 0x55d52d2667c0] error while decoding MB 37 34, bytestream -7", "last_exit_at": "2026-07-01T18:17:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.906, "last_started_at": "2026-07-01T18:17:49+09:00", "pid": 15045, "process": {"exists": false, "pid": 15045}, "restart_count": 106, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 107}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.433, "sample_count": 1187, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 52629, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:19:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T18:19:44+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T18:19:44+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T18:19:44.960288+09:00", "mtime_epoch": 1782897584.9602883, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:19:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.625, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.014, "pushed": 4.014, "read": 4.014}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.66, "count": 119, "max_ms": 12.663, "p95_ms": 10.728}, "gstreamer_push": {"avg_ms": 0.948, "count": 120, "max_ms": 3.236, "p95_ms": 1.777}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.646, "count": 119, "max_ms": 72.478, "p95_ms": 66.741}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.832, "count": 119, "max_ms": 93.939, "p95_ms": 88.512}, "yolo11_640_fire_11class": {"avg_ms": 53.929, "count": 119, "max_ms": 73.084, "p95_ms": 69.852}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.721, "count": 119, "max_ms": 95.8, "p95_ms": 88.555}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.313, "count": 120, "max_ms": 341.5, "p95_ms": 275.485}, "read_interval": {"avg_ms": 250.756, "count": 120, "max_ms": 351.457, "p95_ms": 284.582}, "triton_wait": {"avg_ms": 69.856, "count": 119, "max_ms": 95.814, "p95_ms": 89.023}, "visualize": {"avg_ms": 11.226, "count": 119, "max_ms": 15.487, "p95_ms": 12.609}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 438}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4007.508, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:19:45+09:00", "last_error_summary": "[h264 @ 0x55d52d2667c0] error while decoding MB 37 34, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:19:45+09:00", "last_error_summary": "[h264 @ 0x55d52d2667c0] error while decoding MB 37 34, bytestream -7", "last_exit_at": "2026-07-01T18:19:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 116.081, "last_started_at": "2026-07-01T18:17:49+09:00", "pid": 15045, "process": {"exists": false, "pid": 15045}, "restart_count": 106, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 107}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.433, "sample_count": 1188, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 52630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:19:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:19:45+09:00", "last_frame_read_age_sec": 0.228, "last_frame_read_at": "2026-07-01T18:19:44+09:00", "mjpeg": {"age_sec": 0.219, "exists": true, "mtime": "2026-07-01T18:19:44.960288+09:00", "mtime_epoch": 1782897584.9602883, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:19:45+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 881.656, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.026, "pushed": 4.026, "read": 3.992}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.667, "count": 120, "max_ms": 12.663, "p95_ms": 10.697}, "gstreamer_push": {"avg_ms": 0.948, "count": 120, "max_ms": 3.236, "p95_ms": 1.777}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.562, "count": 120, "max_ms": 72.478, "p95_ms": 64.831}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.764, "count": 120, "max_ms": 93.939, "p95_ms": 88.274}, "yolo11_640_fire_11class": {"avg_ms": 53.855, "count": 120, "max_ms": 73.084, "p95_ms": 69.605}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.628, "count": 120, "max_ms": 95.8, "p95_ms": 88.23}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.109, "count": 120, "max_ms": 341.5, "p95_ms": 275.485}, "read_interval": {"avg_ms": 250.694, "count": 119, "max_ms": 351.457, "p95_ms": 284.854}, "triton_wait": {"avg_ms": 69.779, "count": 120, "max_ms": 95.814, "p95_ms": 88.884}, "visualize": {"avg_ms": 11.223, "count": 120, "max_ms": 15.487, "p95_ms": 12.581}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.809, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 438}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4007.621, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:19:45+09:00", "last_error_summary": "[h264 @ 0x55d52d2667c0] error while decoding MB 37 34, bytestream -7", "last_exit_at": "2026-07-01T18:19:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 116.085, "last_started_at": "2026-07-01T18:17:49+09:00", "pid": 15045, "process": {"exists": false, "pid": 15045}, "restart_count": 106, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 107}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.433, "sample_count": 1188, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 52630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:19:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:19:45+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T18:19:44+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T18:19:44.960288+09:00", "mtime_epoch": 1782897584.9602883, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185125}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:19:45+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 881.656, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.025, "pushed": 4.025, "read": 3.992}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.667, "count": 120, "max_ms": 12.663, "p95_ms": 10.697}, "gstreamer_push": {"avg_ms": 0.948, "count": 120, "max_ms": 3.236, "p95_ms": 1.777}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.562, "count": 120, "max_ms": 72.478, "p95_ms": 64.831}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.764, "count": 120, "max_ms": 93.939, "p95_ms": 88.274}, "yolo11_640_fire_11class": {"avg_ms": 53.855, "count": 120, "max_ms": 73.084, "p95_ms": 69.605}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.628, "count": 120, "max_ms": 95.8, "p95_ms": 88.23}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.109, "count": 120, "max_ms": 341.5, "p95_ms": 275.485}, "read_interval": {"avg_ms": 250.694, "count": 119, "max_ms": 351.457, "p95_ms": 284.854}, "triton_wait": {"avg_ms": 69.779, "count": 120, "max_ms": 95.814, "p95_ms": 88.884}, "visualize": {"avg_ms": 11.223, "count": 120, "max_ms": 15.487, "p95_ms": 12.581}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.813, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 438}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4115.598, "limit_mb": 8796093022207.996}, "detail": {"pid": 15152, "restart_count": 107, "start_count": 108}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 92, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:19:45+09:00", "last_error_summary": "[h264 @ 0x55d52d2667c0] error while decoding MB 37 34, bytestream -7", "last_exit_at": "2026-07-01T18:19:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:19:47+09:00", "pid": 15152, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.785, "name": "ffmpeg", "pid": 15152, "rss_mb": 141.785, "state": "S (sleeping)", "threads": 106}, "restart_count": 107, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 108}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.613, "sample_count": 1169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 52630, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:19:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.394, "last_frame_pushed_at": "2026-07-01T18:19:45+09:00", "last_frame_read_age_sec": 4.62, "last_frame_read_at": "2026-07-01T18:19:44+09:00", "mjpeg": {"age_sec": 4.378, "exists": true, "mtime": "2026-07-01T18:19:45.192292+09:00", "mtime_epoch": 1782897585.192292, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185412}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:19:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 881.656, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.48, "pushed": 3.48, "read": 3.447}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.567, "count": 103, "max_ms": 12.647, "p95_ms": 10.528}, "gstreamer_push": {"avg_ms": 0.933, "count": 104, "max_ms": 3.236, "p95_ms": 1.774}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.436, "count": 103, "max_ms": 72.478, "p95_ms": 63.66}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.867, "count": 103, "max_ms": 93.939, "p95_ms": 88.512}, "yolo11_640_fire_11class": {"avg_ms": 53.777, "count": 103, "max_ms": 73.084, "p95_ms": 69.852}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.43, "count": 103, "max_ms": 95.8, "p95_ms": 87.719}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.407, "count": 104, "max_ms": 341.5, "p95_ms": 271.671}, "read_interval": {"avg_ms": 247.92, "count": 103, "max_ms": 351.457, "p95_ms": 280.836}, "triton_wait": {"avg_ms": 69.609, "count": 103, "max_ms": 95.814, "p95_ms": 88.698}, "visualize": {"avg_ms": 11.099, "count": 103, "max_ms": 15.191, "p95_ms": 12.451}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 442}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4003.855, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f1cb730940] error while decoding MB 74 31, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 93, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:21:38+09:00", "last_error_summary": "[h264 @ 0x55f1cb730940] error while decoding MB 74 31, bytestream -5", "last_exit_at": "2026-07-01T18:19:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.461, "last_started_at": "2026-07-01T18:19:47+09:00", "pid": 15152, "process": {"exists": false, "pid": 15152}, "restart_count": 107, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 108}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.617, "sample_count": 1180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53066, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:21:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.045, "last_frame_pushed_at": "2026-07-01T18:21:37+09:00", "last_frame_read_age_sec": 0.037, "last_frame_read_at": "2026-07-01T18:21:37+09:00", "mjpeg": {"age_sec": 0.03, "exists": true, "mtime": "2026-07-01T18:21:38.001026+09:00", "mtime_epoch": 1782897698.0010257, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185050}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:21:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.289, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.982, "pushed": 3.982, "read": 3.982}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.646, "count": 118, "max_ms": 17.174, "p95_ms": 10.856}, "gstreamer_push": {"avg_ms": 0.97, "count": 119, "max_ms": 2.998, "p95_ms": 1.866}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.335, "count": 118, "max_ms": 69.569, "p95_ms": 67.033}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.3, "count": 118, "max_ms": 92.894, "p95_ms": 87.678}, "yolo11_640_fire_11class": {"avg_ms": 53.638, "count": 118, "max_ms": 71.862, "p95_ms": 69.01}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.607, "count": 118, "max_ms": 92.471, "p95_ms": 89.98}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.895, "count": 119, "max_ms": 318.41, "p95_ms": 272.925}, "read_interval": {"avg_ms": 252.69, "count": 119, "max_ms": 329.834, "p95_ms": 282.735}, "triton_wait": {"avg_ms": 69.388, "count": 118, "max_ms": 92.906, "p95_ms": 90.177}, "visualize": {"avg_ms": 11.354, "count": 118, "max_ms": 15.506, "p95_ms": 12.447}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.886, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 442}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4002.648, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:21:38+09:00", "last_error_summary": "[h264 @ 0x55f1cb730940] error while decoding MB 74 31, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 93, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:21:38+09:00", "last_error_summary": "[h264 @ 0x55f1cb730940] error while decoding MB 74 31, bytestream -5", "last_exit_at": "2026-07-01T18:21:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.739, "last_started_at": "2026-07-01T18:19:47+09:00", "pid": 15152, "process": {"exists": false, "pid": 15152}, "restart_count": 107, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 108}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.654, "sample_count": 1180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53067, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:21:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:21:38+09:00", "last_frame_read_age_sec": 0.316, "last_frame_read_at": "2026-07-01T18:21:37+09:00", "mjpeg": {"age_sec": 0.308, "exists": true, "mtime": "2026-07-01T18:21:38.001026+09:00", "mtime_epoch": 1782897698.0010257, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185050}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:21:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.078, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.978, "pushed": 3.978, "read": 3.944}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.654, "count": 118, "max_ms": 17.174, "p95_ms": 10.856}, "gstreamer_push": {"avg_ms": 0.977, "count": 119, "max_ms": 2.998, "p95_ms": 1.866}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.449, "count": 118, "max_ms": 70.009, "p95_ms": 68.334}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.468, "count": 118, "max_ms": 92.894, "p95_ms": 87.844}, "yolo11_640_fire_11class": {"avg_ms": 53.775, "count": 118, "max_ms": 71.862, "p95_ms": 69.038}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.783, "count": 118, "max_ms": 92.471, "p95_ms": 89.98}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.637, "count": 119, "max_ms": 318.41, "p95_ms": 284.695}, "read_interval": {"avg_ms": 252.86, "count": 118, "max_ms": 329.834, "p95_ms": 282.735}, "triton_wait": {"avg_ms": 69.564, "count": 118, "max_ms": 92.906, "p95_ms": 90.177}, "visualize": {"avg_ms": 11.351, "count": 118, "max_ms": 15.506, "p95_ms": 12.447}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 442}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4002.641, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 93, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:21:38+09:00", "last_error_summary": "[h264 @ 0x55f1cb730940] error while decoding MB 74 31, bytestream -5", "last_exit_at": "2026-07-01T18:21:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.743, "last_started_at": "2026-07-01T18:19:47+09:00", "pid": 15152, "process": {"exists": false, "pid": 15152}, "restart_count": 107, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 108}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.654, "sample_count": 1180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53067, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:21:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:21:38+09:00", "last_frame_read_age_sec": 0.319, "last_frame_read_at": "2026-07-01T18:21:37+09:00", "mjpeg": {"age_sec": 0.311, "exists": true, "mtime": "2026-07-01T18:21:38.001026+09:00", "mtime_epoch": 1782897698.0010257, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185050}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:21:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.078, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.977, "pushed": 3.977, "read": 3.944}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.654, "count": 118, "max_ms": 17.174, "p95_ms": 10.856}, "gstreamer_push": {"avg_ms": 0.977, "count": 119, "max_ms": 2.998, "p95_ms": 1.866}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.449, "count": 118, "max_ms": 70.009, "p95_ms": 68.334}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.468, "count": 118, "max_ms": 92.894, "p95_ms": 87.844}, "yolo11_640_fire_11class": {"avg_ms": 53.775, "count": 118, "max_ms": 71.862, "p95_ms": 69.038}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.783, "count": 118, "max_ms": 92.471, "p95_ms": 89.98}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.637, "count": 119, "max_ms": 318.41, "p95_ms": 284.695}, "read_interval": {"avg_ms": 252.86, "count": 118, "max_ms": 329.834, "p95_ms": 282.735}, "triton_wait": {"avg_ms": 69.564, "count": 118, "max_ms": 92.906, "p95_ms": 90.177}, "visualize": {"avg_ms": 11.351, "count": 118, "max_ms": 15.506, "p95_ms": 12.447}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 442}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4109.402, "limit_mb": 8796093022207.996}, "detail": {"pid": 15259, "restart_count": 108, "start_count": 109}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 93, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:21:38+09:00", "last_error_summary": "[h264 @ 0x55f1cb730940] error while decoding MB 74 31, bytestream -5", "last_exit_at": "2026-07-01T18:21:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:21:40+09:00", "pid": 15259, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.191, "name": "ffmpeg", "pid": 15259, "rss_mb": 139.191, "state": "S (sleeping)", "threads": 106}, "restart_count": 108, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 109}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.654, "sample_count": 1164, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53067, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:21:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.288, "last_frame_pushed_at": "2026-07-01T18:21:38+09:00", "last_frame_read_age_sec": 4.601, "last_frame_read_at": "2026-07-01T18:21:37+09:00", "mjpeg": {"age_sec": 4.274, "exists": true, "mtime": "2026-07-01T18:21:38.321040+09:00", "mtime_epoch": 1782897698.3210397, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185347}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:21:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 880.078, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.418, "pushed": 3.418, "read": 3.385}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.64, "count": 102, "max_ms": 17.174, "p95_ms": 10.648}, "gstreamer_push": {"avg_ms": 0.975, "count": 102, "max_ms": 2.998, "p95_ms": 1.812}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.45, "count": 102, "max_ms": 70.009, "p95_ms": 68.334}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.122, "count": 102, "max_ms": 92.894, "p95_ms": 87.844}, "yolo11_640_fire_11class": {"avg_ms": 54.196, "count": 102, "max_ms": 71.862, "p95_ms": 69.038}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.33, "count": 102, "max_ms": 92.471, "p95_ms": 89.98}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.144, "count": 102, "max_ms": 318.41, "p95_ms": 284.695}, "read_interval": {"avg_ms": 252.266, "count": 101, "max_ms": 329.834, "p95_ms": 282.735}, "triton_wait": {"avg_ms": 70.194, "count": 102, "max_ms": 92.906, "p95_ms": 90.177}, "visualize": {"avg_ms": 11.259, "count": 102, "max_ms": 15.506, "p95_ms": 12.207}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.841, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 446}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4012.5, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c10ccd5e80] error while decoding MB 74 34, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 96, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:23:29+09:00", "last_error_summary": "[h264 @ 0x55c10ccd5e80] error while decoding MB 74 34, bytestream -9", "last_exit_at": "2026-07-01T18:21:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.464, "last_started_at": "2026-07-01T18:21:40+09:00", "pid": 15259, "process": {"exists": false, "pid": 15259}, "restart_count": 108, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 109}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.822, "sample_count": 1199, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53521, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:23:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T18:23:29+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T18:23:29+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T18:23:29.017580+09:00", "mtime_epoch": 1782897809.0175803, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178520}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:23:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.844, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.291, "pushed": 4.291, "read": 4.291}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.977, "count": 128, "max_ms": 12.471, "p95_ms": 10.373}, "gstreamer_push": {"avg_ms": 0.938, "count": 128, "max_ms": 2.932, "p95_ms": 1.754}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.435, "count": 128, "max_ms": 67.537, "p95_ms": 60.859}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.638, "count": 128, "max_ms": 91.171, "p95_ms": 87.709}, "yolo11_640_fire_11class": {"avg_ms": 52.987, "count": 128, "max_ms": 70.295, "p95_ms": 66.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.387, "count": 128, "max_ms": 90.797, "p95_ms": 87.835}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.588, "count": 128, "max_ms": 338.079, "p95_ms": 269.341}, "read_interval": {"avg_ms": 234.262, "count": 128, "max_ms": 347.121, "p95_ms": 278.526}, "triton_wait": {"avg_ms": 68.348, "count": 128, "max_ms": 91.184, "p95_ms": 87.848}, "visualize": {"avg_ms": 10.437, "count": 128, "max_ms": 13.829, "p95_ms": 12.048}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.83, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 446}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4011.469, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:23:29+09:00", "last_error_summary": "[h264 @ 0x55c10ccd5e80] error while decoding MB 74 34, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 96, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:23:29+09:00", "last_error_summary": "[h264 @ 0x55c10ccd5e80] error while decoding MB 74 34, bytestream -9", "last_exit_at": "2026-07-01T18:23:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.654, "last_started_at": "2026-07-01T18:21:40+09:00", "pid": 15259, "process": {"exists": false, "pid": 15259}, "restart_count": 108, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 109}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.822, "sample_count": 1199, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53522, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:23:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:23:29+09:00", "last_frame_read_age_sec": 0.239, "last_frame_read_at": "2026-07-01T18:23:29+09:00", "mjpeg": {"age_sec": 0.23, "exists": true, "mtime": "2026-07-01T18:23:29.017580+09:00", "mtime_epoch": 1782897809.0175803, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178520}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:23:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.844, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.295, "pushed": 4.295, "read": 4.262}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.983, "count": 128, "max_ms": 12.471, "p95_ms": 10.373}, "gstreamer_push": {"avg_ms": 0.938, "count": 128, "max_ms": 2.932, "p95_ms": 1.754}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.371, "count": 128, "max_ms": 67.537, "p95_ms": 60.859}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.615, "count": 128, "max_ms": 91.171, "p95_ms": 87.709}, "yolo11_640_fire_11class": {"avg_ms": 52.982, "count": 128, "max_ms": 70.295, "p95_ms": 66.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.388, "count": 128, "max_ms": 90.797, "p95_ms": 87.835}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.822, "count": 128, "max_ms": 338.079, "p95_ms": 269.341}, "read_interval": {"avg_ms": 234.419, "count": 127, "max_ms": 347.121, "p95_ms": 278.526}, "triton_wait": {"avg_ms": 68.331, "count": 128, "max_ms": 91.184, "p95_ms": 87.848}, "visualize": {"avg_ms": 10.448, "count": 128, "max_ms": 13.829, "p95_ms": 12.048}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.799, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 446}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4011.461, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 96, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:23:29+09:00", "last_error_summary": "[h264 @ 0x55c10ccd5e80] error while decoding MB 74 34, bytestream -9", "last_exit_at": "2026-07-01T18:23:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.657, "last_started_at": "2026-07-01T18:21:40+09:00", "pid": 15259, "process": {"exists": false, "pid": 15259}, "restart_count": 108, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 109}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.822, "sample_count": 1199, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53522, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:23:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:23:29+09:00", "last_frame_read_age_sec": 0.243, "last_frame_read_at": "2026-07-01T18:23:29+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-01T18:23:29.017580+09:00", "mtime_epoch": 1782897809.0175803, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178520}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:23:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.844, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.295, "pushed": 4.295, "read": 4.261}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.983, "count": 128, "max_ms": 12.471, "p95_ms": 10.373}, "gstreamer_push": {"avg_ms": 0.938, "count": 128, "max_ms": 2.932, "p95_ms": 1.754}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.371, "count": 128, "max_ms": 67.537, "p95_ms": 60.859}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.615, "count": 128, "max_ms": 91.171, "p95_ms": 87.709}, "yolo11_640_fire_11class": {"avg_ms": 52.982, "count": 128, "max_ms": 70.295, "p95_ms": 66.932}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.388, "count": 128, "max_ms": 90.797, "p95_ms": 87.835}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.822, "count": 128, "max_ms": 338.079, "p95_ms": 269.341}, "read_interval": {"avg_ms": 234.419, "count": 127, "max_ms": 347.121, "p95_ms": 278.526}, "triton_wait": {"avg_ms": 68.331, "count": 128, "max_ms": 91.184, "p95_ms": 87.848}, "visualize": {"avg_ms": 10.448, "count": 128, "max_ms": 13.829, "p95_ms": 12.048}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.802, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 446}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4119.363, "limit_mb": 8796093022207.996}, "detail": {"pid": 15366, "restart_count": 109, "start_count": 110}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 94, "h264_decode_error_count": 96, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:23:29+09:00", "last_error_summary": "[h264 @ 0x55c10ccd5e80] error while decoding MB 74 34, bytestream -9", "last_exit_at": "2026-07-01T18:23:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:23:31+09:00", "pid": 15366, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.027, "name": "ffmpeg", "pid": 15366, "rss_mb": 139.027, "state": "S (sleeping)", "threads": 106}, "restart_count": 109, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 110}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.822, "sample_count": 1180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53522, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:23:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.521, "last_frame_pushed_at": "2026-07-01T18:23:29+09:00", "last_frame_read_age_sec": 4.757, "last_frame_read_at": "2026-07-01T18:23:29+09:00", "mjpeg": {"age_sec": 4.504, "exists": true, "mtime": "2026-07-01T18:23:29.261590+09:00", "mtime_epoch": 1782897809.2615898, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178526}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:23:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.844, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.618, "pushed": 3.618, "read": 3.585}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.121, "count": 108, "max_ms": 12.471, "p95_ms": 10.387}, "gstreamer_push": {"avg_ms": 0.948, "count": 108, "max_ms": 2.932, "p95_ms": 1.798}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.853, "count": 108, "max_ms": 67.444, "p95_ms": 60.363}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.328, "count": 108, "max_ms": 89.271, "p95_ms": 84.051}, "yolo11_640_fire_11class": {"avg_ms": 52.764, "count": 108, "max_ms": 70.295, "p95_ms": 66.814}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.26, "count": 108, "max_ms": 90.797, "p95_ms": 87.384}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.001, "count": 108, "max_ms": 338.079, "p95_ms": 270.437}, "read_interval": {"avg_ms": 236.598, "count": 107, "max_ms": 347.121, "p95_ms": 281.71}, "triton_wait": {"avg_ms": 68.116, "count": 108, "max_ms": 90.81, "p95_ms": 87.842}, "visualize": {"avg_ms": 10.534, "count": 108, "max_ms": 13.829, "p95_ms": 12.094}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.849, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 449}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4012.691, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5588e97be1c0] error while decoding MB 30 36, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 97, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:25:22+09:00", "last_error_summary": "[h264 @ 0x5588e97be1c0] error while decoding MB 30 36, bytestream -9", "last_exit_at": "2026-07-01T18:23:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.466, "last_started_at": "2026-07-01T18:23:31+09:00", "pid": 15366, "process": {"exists": false, "pid": 15366}, "restart_count": 109, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 110}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.369, "sample_count": 1186, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53949, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:25:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T18:25:22+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T18:25:22+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T18:25:22.193735+09:00", "mtime_epoch": 1782897922.193735, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178035}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:25:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.566, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.015, "pushed": 4.015, "read": 4.015}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.908, "count": 119, "max_ms": 14.069, "p95_ms": 11.361}, "gstreamer_push": {"avg_ms": 0.975, "count": 120, "max_ms": 3.044, "p95_ms": 1.847}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.674, "count": 119, "max_ms": 70.596, "p95_ms": 64.309}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.037, "count": 119, "max_ms": 90.558, "p95_ms": 87.876}, "yolo11_640_fire_11class": {"avg_ms": 53.055, "count": 119, "max_ms": 72.752, "p95_ms": 69.616}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.478, "count": 119, "max_ms": 92.181, "p95_ms": 88.363}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.802, "count": 120, "max_ms": 351.168, "p95_ms": 270.681}, "read_interval": {"avg_ms": 250.487, "count": 120, "max_ms": 362.66, "p95_ms": 280.373}, "triton_wait": {"avg_ms": 68.183, "count": 119, "max_ms": 92.193, "p95_ms": 89.39}, "visualize": {"avg_ms": 11.432, "count": 119, "max_ms": 16.336, "p95_ms": 13.696}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 449}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4011.867, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:25:22+09:00", "last_error_summary": "[h264 @ 0x5588e97be1c0] error while decoding MB 30 36, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 97, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:25:22+09:00", "last_error_summary": "[h264 @ 0x5588e97be1c0] error while decoding MB 30 36, bytestream -9", "last_exit_at": "2026-07-01T18:25:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.662, "last_started_at": "2026-07-01T18:23:31+09:00", "pid": 15366, "process": {"exists": false, "pid": 15366}, "restart_count": 109, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 110}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.369, "sample_count": 1186, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53950, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:25:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:25:22+09:00", "last_frame_read_age_sec": 0.248, "last_frame_read_at": "2026-07-01T18:25:22+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-01T18:25:22.193735+09:00", "mtime_epoch": 1782897922.193735, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178035}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:25:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.566, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.022, "pushed": 4.022, "read": 3.989}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.91, "count": 120, "max_ms": 14.069, "p95_ms": 11.224}, "gstreamer_push": {"avg_ms": 0.975, "count": 120, "max_ms": 3.044, "p95_ms": 1.847}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.62, "count": 120, "max_ms": 70.596, "p95_ms": 62.343}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.987, "count": 120, "max_ms": 90.558, "p95_ms": 87.414}, "yolo11_640_fire_11class": {"avg_ms": 53.002, "count": 120, "max_ms": 72.752, "p95_ms": 68.824}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.418, "count": 120, "max_ms": 92.181, "p95_ms": 88.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.976, "count": 120, "max_ms": 351.168, "p95_ms": 270.681}, "read_interval": {"avg_ms": 250.627, "count": 119, "max_ms": 362.66, "p95_ms": 280.738}, "triton_wait": {"avg_ms": 68.124, "count": 120, "max_ms": 92.193, "p95_ms": 88.806}, "visualize": {"avg_ms": 11.439, "count": 120, "max_ms": 16.336, "p95_ms": 13.149}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.836, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 449}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4011.859, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 97, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:25:22+09:00", "last_error_summary": "[h264 @ 0x5588e97be1c0] error while decoding MB 30 36, bytestream -9", "last_exit_at": "2026-07-01T18:25:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.666, "last_started_at": "2026-07-01T18:23:31+09:00", "pid": 15366, "process": {"exists": false, "pid": 15366}, "restart_count": 109, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 110}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.369, "sample_count": 1186, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53950, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:25:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:25:22+09:00", "last_frame_read_age_sec": 0.252, "last_frame_read_at": "2026-07-01T18:25:22+09:00", "mjpeg": {"age_sec": 0.236, "exists": true, "mtime": "2026-07-01T18:25:22.193735+09:00", "mtime_epoch": 1782897922.193735, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178035}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:25:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.566, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.022, "pushed": 4.022, "read": 3.988}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.91, "count": 120, "max_ms": 14.069, "p95_ms": 11.224}, "gstreamer_push": {"avg_ms": 0.975, "count": 120, "max_ms": 3.044, "p95_ms": 1.847}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.62, "count": 120, "max_ms": 70.596, "p95_ms": 62.343}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.987, "count": 120, "max_ms": 90.558, "p95_ms": 87.414}, "yolo11_640_fire_11class": {"avg_ms": 53.002, "count": 120, "max_ms": 72.752, "p95_ms": 68.824}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.418, "count": 120, "max_ms": 92.181, "p95_ms": 88.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.976, "count": 120, "max_ms": 351.168, "p95_ms": 270.681}, "read_interval": {"avg_ms": 250.627, "count": 119, "max_ms": 362.66, "p95_ms": 280.738}, "triton_wait": {"avg_ms": 68.124, "count": 120, "max_ms": 92.193, "p95_ms": 88.806}, "visualize": {"avg_ms": 11.439, "count": 120, "max_ms": 16.336, "p95_ms": 13.149}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.839, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 449}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4118.223, "limit_mb": 8796093022207.996}, "detail": {"pid": 15473, "restart_count": 110, "start_count": 111}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 95, "h264_decode_error_count": 97, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:25:22+09:00", "last_error_summary": "[h264 @ 0x5588e97be1c0] error while decoding MB 30 36, bytestream -9", "last_exit_at": "2026-07-01T18:25:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:25:25+09:00", "pid": 15473, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.027, "name": "ffmpeg", "pid": 15473, "rss_mb": 136.027, "state": "S (sleeping)", "threads": 106}, "restart_count": 110, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 111}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.467, "sample_count": 1167, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 53950, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:25:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.731, "last_frame_pushed_at": "2026-07-01T18:25:22+09:00", "last_frame_read_age_sec": 4.977, "last_frame_read_at": "2026-07-01T18:25:22+09:00", "mjpeg": {"age_sec": 4.713, "exists": true, "mtime": "2026-07-01T18:25:22.441744+09:00", "mtime_epoch": 1782897922.4417436, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178069}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:25:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.566, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.38, "pushed": 3.38, "read": 3.346}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.904, "count": 100, "max_ms": 14.069, "p95_ms": 11.361}, "gstreamer_push": {"avg_ms": 0.982, "count": 101, "max_ms": 3.044, "p95_ms": 1.847}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.832, "count": 100, "max_ms": 70.596, "p95_ms": 64.309}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.464, "count": 100, "max_ms": 89.378, "p95_ms": 87.167}, "yolo11_640_fire_11class": {"avg_ms": 52.842, "count": 100, "max_ms": 72.752, "p95_ms": 68.713}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.31, "count": 100, "max_ms": 92.181, "p95_ms": 88.331}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.904, "count": 101, "max_ms": 351.168, "p95_ms": 270.681}, "read_interval": {"avg_ms": 251.557, "count": 100, "max_ms": 362.66, "p95_ms": 280.373}, "triton_wait": {"avg_ms": 67.793, "count": 100, "max_ms": 92.193, "p95_ms": 88.343}, "visualize": {"avg_ms": 11.523, "count": 100, "max_ms": 16.336, "p95_ms": 13.696}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 453}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4016.777, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e4432bd540] error while decoding MB 16 41, bytestream -16", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 96, "h264_decode_error_count": 98, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:27:14+09:00", "last_error_summary": "[h264 @ 0x55e4432bd540] error while decoding MB 16 41, bytestream -16", "last_exit_at": "2026-07-01T18:25:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.948, "last_started_at": "2026-07-01T18:25:25+09:00", "pid": 15473, "process": {"exists": false, "pid": 15473}, "restart_count": 110, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 111}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.59, "sample_count": 1165, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 54371, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:27:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T18:27:14+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T18:27:14+09:00", "mjpeg": {"age_sec": 0.036, "exists": true, "mtime": "2026-07-01T18:27:14.065452+09:00", "mtime_epoch": 1782898034.065452, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180240}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:27:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.836, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.099, "pushed": 4.099, "read": 4.099}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.281, "count": 122, "max_ms": 15.723, "p95_ms": 10.167}, "gstreamer_push": {"avg_ms": 0.918, "count": 122, "max_ms": 2.662, "p95_ms": 1.794}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.957, "count": 122, "max_ms": 71.504, "p95_ms": 61.675}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.675, "count": 122, "max_ms": 89.807, "p95_ms": 84.081}, "yolo11_640_fire_11class": {"avg_ms": 53.132, "count": 122, "max_ms": 73.062, "p95_ms": 68.108}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.938, "count": 122, "max_ms": 93.774, "p95_ms": 86.313}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.098, "count": 122, "max_ms": 311.29, "p95_ms": 270.399}, "read_interval": {"avg_ms": 245.631, "count": 122, "max_ms": 322.053, "p95_ms": 279.296}, "triton_wait": {"avg_ms": 67.172, "count": 122, "max_ms": 93.785, "p95_ms": 87.63}, "visualize": {"avg_ms": 11.208, "count": 122, "max_ms": 15.918, "p95_ms": 12.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.764, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 453}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4015.598, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:27:14+09:00", "last_error_summary": "[h264 @ 0x55e4432bd540] error while decoding MB 16 41, bytestream -16", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 96, "h264_decode_error_count": 98, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:27:14+09:00", "last_error_summary": "[h264 @ 0x55e4432bd540] error while decoding MB 16 41, bytestream -16", "last_exit_at": "2026-07-01T18:27:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.153, "last_started_at": "2026-07-01T18:25:25+09:00", "pid": 15473, "process": {"exists": false, "pid": 15473}, "restart_count": 110, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 111}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.59, "sample_count": 1165, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 54372, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:27:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:27:14+09:00", "last_frame_read_age_sec": 0.255, "last_frame_read_at": "2026-07-01T18:27:14+09:00", "mjpeg": {"age_sec": 0.241, "exists": true, "mtime": "2026-07-01T18:27:14.065452+09:00", "mtime_epoch": 1782898034.065452, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180240}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:27:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.836, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.104, "pushed": 4.104, "read": 4.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.275, "count": 122, "max_ms": 15.723, "p95_ms": 10.167}, "gstreamer_push": {"avg_ms": 0.917, "count": 123, "max_ms": 2.662, "p95_ms": 1.794}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.046, "count": 122, "max_ms": 71.504, "p95_ms": 61.675}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.794, "count": 122, "max_ms": 89.807, "p95_ms": 86.078}, "yolo11_640_fire_11class": {"avg_ms": 53.261, "count": 122, "max_ms": 73.062, "p95_ms": 68.47}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.111, "count": 122, "max_ms": 93.774, "p95_ms": 87.056}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.221, "count": 123, "max_ms": 311.29, "p95_ms": 270.399}, "read_interval": {"avg_ms": 245.631, "count": 122, "max_ms": 322.053, "p95_ms": 279.296}, "triton_wait": {"avg_ms": 67.327, "count": 122, "max_ms": 93.785, "p95_ms": 87.709}, "visualize": {"avg_ms": 11.196, "count": 122, "max_ms": 15.918, "p95_ms": 12.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 453}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4015.445, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 96, "h264_decode_error_count": 98, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:27:14+09:00", "last_error_summary": "[h264 @ 0x55e4432bd540] error while decoding MB 16 41, bytestream -16", "last_exit_at": "2026-07-01T18:27:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.156, "last_started_at": "2026-07-01T18:25:25+09:00", "pid": 15473, "process": {"exists": false, "pid": 15473}, "restart_count": 110, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 111}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.59, "sample_count": 1165, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 54372, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:27:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:27:14+09:00", "last_frame_read_age_sec": 0.259, "last_frame_read_at": "2026-07-01T18:27:14+09:00", "mjpeg": {"age_sec": 0.245, "exists": true, "mtime": "2026-07-01T18:27:14.065452+09:00", "mtime_epoch": 1782898034.065452, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180240}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:27:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.836, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.104, "pushed": 4.104, "read": 4.07}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.275, "count": 122, "max_ms": 15.723, "p95_ms": 10.167}, "gstreamer_push": {"avg_ms": 0.917, "count": 123, "max_ms": 2.662, "p95_ms": 1.794}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.046, "count": 122, "max_ms": 71.504, "p95_ms": 61.675}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.794, "count": 122, "max_ms": 89.807, "p95_ms": 86.078}, "yolo11_640_fire_11class": {"avg_ms": 53.261, "count": 122, "max_ms": 73.062, "p95_ms": 68.47}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.111, "count": 122, "max_ms": 93.774, "p95_ms": 87.056}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.221, "count": 123, "max_ms": 311.29, "p95_ms": 270.399}, "read_interval": {"avg_ms": 245.631, "count": 122, "max_ms": 322.053, "p95_ms": 279.296}, "triton_wait": {"avg_ms": 67.327, "count": 122, "max_ms": 93.785, "p95_ms": 87.709}, "visualize": {"avg_ms": 11.196, "count": 122, "max_ms": 15.918, "p95_ms": 12.54}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 453}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4126.012, "limit_mb": 8796093022207.996}, "detail": {"pid": 15580, "restart_count": 111, "start_count": 112}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 96, "h264_decode_error_count": 98, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:27:14+09:00", "last_error_summary": "[h264 @ 0x55e4432bd540] error while decoding MB 16 41, bytestream -16", "last_exit_at": "2026-07-01T18:27:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:27:17+09:00", "pid": 15580, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.543, "name": "ffmpeg", "pid": 15580, "rss_mb": 141.543, "state": "S (sleeping)", "threads": 106}, "restart_count": 111, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 112}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.853, "sample_count": 1145, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 54372, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:27:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.961, "last_frame_pushed_at": "2026-07-01T18:27:14+09:00", "last_frame_read_age_sec": 5.213, "last_frame_read_at": "2026-07-01T18:27:14+09:00", "mjpeg": {"age_sec": 4.943, "exists": true, "mtime": "2026-07-01T18:27:14.321460+09:00", "mtime_epoch": 1782898034.3214602, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179950}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:27:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 886.836, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.446, "pushed": 3.446, "read": 3.413}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.201, "count": 102, "max_ms": 11.527, "p95_ms": 9.949}, "gstreamer_push": {"avg_ms": 0.918, "count": 103, "max_ms": 2.662, "p95_ms": 1.794}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.251, "count": 102, "max_ms": 71.504, "p95_ms": 61.289}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.847, "count": 102, "max_ms": 87.618, "p95_ms": 84.081}, "yolo11_640_fire_11class": {"avg_ms": 53.318, "count": 102, "max_ms": 73.062, "p95_ms": 68.108}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.146, "count": 102, "max_ms": 93.774, "p95_ms": 86.313}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.874, "count": 103, "max_ms": 311.29, "p95_ms": 264.256}, "read_interval": {"avg_ms": 244.233, "count": 102, "max_ms": 322.053, "p95_ms": 274.077}, "triton_wait": {"avg_ms": 67.378, "count": 102, "max_ms": 93.785, "p95_ms": 87.068}, "visualize": {"avg_ms": 11.105, "count": 102, "max_ms": 14.729, "p95_ms": 12.163}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 456}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4012.336, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563b4b8ba300] error while decoding MB 19 9, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 97, "h264_decode_error_count": 99, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:29:09+09:00", "last_error_summary": "[h264 @ 0x563b4b8ba300] error while decoding MB 19 9, bytestream -12", "last_exit_at": "2026-07-01T18:27:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.609, "last_started_at": "2026-07-01T18:27:17+09:00", "pid": 15580, "process": {"exists": false, "pid": 15580}, "restart_count": 111, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 112}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.853, "sample_count": 1152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 54812, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:29:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T18:29:09+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T18:29:09+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T18:29:09.828972+09:00", "mtime_epoch": 1782898149.828972, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181268}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:29:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 884.164, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.979, "pushed": 3.979, "read": 3.979}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.683, "count": 118, "max_ms": 13.795, "p95_ms": 10.867}, "gstreamer_push": {"avg_ms": 0.903, "count": 119, "max_ms": 2.119, "p95_ms": 1.063}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.622, "count": 118, "max_ms": 72.374, "p95_ms": 65.807}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.297, "count": 118, "max_ms": 90.024, "p95_ms": 88.912}, "yolo11_640_fire_11class": {"avg_ms": 55.686, "count": 118, "max_ms": 74.07, "p95_ms": 70.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.432, "count": 118, "max_ms": 92.606, "p95_ms": 89.291}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.244, "count": 119, "max_ms": 300.263, "p95_ms": 274.754}, "read_interval": {"avg_ms": 252.901, "count": 119, "max_ms": 311.865, "p95_ms": 284.024}, "triton_wait": {"avg_ms": 70.714, "count": 118, "max_ms": 92.619, "p95_ms": 89.964}, "visualize": {"avg_ms": 11.403, "count": 118, "max_ms": 16.595, "p95_ms": 12.749}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 456}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4011.117, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:29:09+09:00", "last_error_summary": "[h264 @ 0x563b4b8ba300] error while decoding MB 19 9, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 97, "h264_decode_error_count": 99, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:29:09+09:00", "last_error_summary": "[h264 @ 0x563b4b8ba300] error while decoding MB 19 9, bytestream -12", "last_exit_at": "2026-07-01T18:29:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.817, "last_started_at": "2026-07-01T18:27:17+09:00", "pid": 15580, "process": {"exists": false, "pid": 15580}, "restart_count": 111, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 112}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.853, "sample_count": 1152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 54813, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:29:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T18:29:10+09:00", "last_frame_read_age_sec": 0.259, "last_frame_read_at": "2026-07-01T18:29:09+09:00", "mjpeg": {"age_sec": 0.252, "exists": true, "mtime": "2026-07-01T18:29:09.828972+09:00", "mtime_epoch": 1782898149.828972, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181268}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:29:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 884.164, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.985, "pushed": 3.985, "read": 3.951}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.679, "count": 118, "max_ms": 13.795, "p95_ms": 10.867}, "gstreamer_push": {"avg_ms": 0.902, "count": 119, "max_ms": 2.119, "p95_ms": 1.063}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.677, "count": 118, "max_ms": 72.374, "p95_ms": 65.807}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.361, "count": 118, "max_ms": 90.024, "p95_ms": 88.912}, "yolo11_640_fire_11class": {"avg_ms": 55.761, "count": 118, "max_ms": 74.07, "p95_ms": 70.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.499, "count": 118, "max_ms": 92.606, "p95_ms": 89.291}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.367, "count": 119, "max_ms": 300.263, "p95_ms": 274.754}, "read_interval": {"avg_ms": 252.94, "count": 118, "max_ms": 311.865, "p95_ms": 284.024}, "triton_wait": {"avg_ms": 70.781, "count": 118, "max_ms": 92.619, "p95_ms": 89.964}, "visualize": {"avg_ms": 11.403, "count": 118, "max_ms": 16.595, "p95_ms": 12.749}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.865, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 456}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4011.234, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 97, "h264_decode_error_count": 99, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:29:09+09:00", "last_error_summary": "[h264 @ 0x563b4b8ba300] error while decoding MB 19 9, bytestream -12", "last_exit_at": "2026-07-01T18:29:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.82, "last_started_at": "2026-07-01T18:27:17+09:00", "pid": 15580, "process": {"exists": false, "pid": 15580}, "restart_count": 111, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 112}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.853, "sample_count": 1152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 54813, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:29:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T18:29:10+09:00", "last_frame_read_age_sec": 0.262, "last_frame_read_at": "2026-07-01T18:29:09+09:00", "mjpeg": {"age_sec": 0.255, "exists": true, "mtime": "2026-07-01T18:29:09.828972+09:00", "mtime_epoch": 1782898149.828972, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181268}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:29:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 884.164, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.984, "pushed": 3.984, "read": 3.951}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.679, "count": 118, "max_ms": 13.795, "p95_ms": 10.867}, "gstreamer_push": {"avg_ms": 0.902, "count": 119, "max_ms": 2.119, "p95_ms": 1.063}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.677, "count": 118, "max_ms": 72.374, "p95_ms": 65.807}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.361, "count": 118, "max_ms": 90.024, "p95_ms": 88.912}, "yolo11_640_fire_11class": {"avg_ms": 55.761, "count": 118, "max_ms": 74.07, "p95_ms": 70.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.499, "count": 118, "max_ms": 92.606, "p95_ms": 89.291}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.367, "count": 119, "max_ms": 300.263, "p95_ms": 274.754}, "read_interval": {"avg_ms": 252.94, "count": 118, "max_ms": 311.865, "p95_ms": 284.024}, "triton_wait": {"avg_ms": 70.781, "count": 118, "max_ms": 92.619, "p95_ms": 89.964}, "visualize": {"avg_ms": 11.403, "count": 118, "max_ms": 16.595, "p95_ms": 12.749}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 456}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4119.836, "limit_mb": 8796093022207.996}, "detail": {"pid": 15687, "restart_count": 112, "start_count": 113}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 97, "h264_decode_error_count": 99, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:29:09+09:00", "last_error_summary": "[h264 @ 0x563b4b8ba300] error while decoding MB 19 9, bytestream -12", "last_exit_at": "2026-07-01T18:29:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:29:12+09:00", "pid": 15687, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.609, "name": "ffmpeg", "pid": 15687, "rss_mb": 139.609, "state": "S (sleeping)", "threads": 106}, "restart_count": 112, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 113}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.754, "sample_count": 1135, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 54813, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:29:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.683, "last_frame_pushed_at": "2026-07-01T18:29:10+09:00", "last_frame_read_age_sec": 4.938, "last_frame_read_at": "2026-07-01T18:29:09+09:00", "mjpeg": {"age_sec": 4.667, "exists": true, "mtime": "2026-07-01T18:29:10.092980+09:00", "mtime_epoch": 1782898150.09298, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181400}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:29:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 884.164, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.391, "pushed": 3.391, "read": 3.358}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.586, "count": 101, "max_ms": 13.795, "p95_ms": 10.867}, "gstreamer_push": {"avg_ms": 0.91, "count": 101, "max_ms": 2.119, "p95_ms": 1.063}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.753, "count": 101, "max_ms": 72.374, "p95_ms": 65.228}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.345, "count": 101, "max_ms": 89.953, "p95_ms": 86.006}, "yolo11_640_fire_11class": {"avg_ms": 55.086, "count": 101, "max_ms": 74.07, "p95_ms": 69.541}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.601, "count": 101, "max_ms": 92.606, "p95_ms": 88.044}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.316, "count": 101, "max_ms": 300.263, "p95_ms": 274.754}, "read_interval": {"avg_ms": 250.876, "count": 100, "max_ms": 311.865, "p95_ms": 284.024}, "triton_wait": {"avg_ms": 69.88, "count": 101, "max_ms": 92.619, "p95_ms": 88.993}, "visualize": {"avg_ms": 11.352, "count": 101, "max_ms": 16.595, "p95_ms": 12.486}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.781, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 460}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4013.102, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5625467ef180] cabac decode of qscale diff failed at 49 44 [h264 @ 0x5625467ef180] error while decoding MB 49 44, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 98, "h264_decode_error_count": 101, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:31:02+09:00", "last_error_summary": "[h264 @ 0x5625467ef180] cabac decode of qscale diff failed at 49 44 [h264 @ 0x5625467ef180] error while decoding MB 49 44, bytestream -5", "last_exit_at": "2026-07-01T18:29:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.121, "last_started_at": "2026-07-01T18:29:12+09:00", "pid": 15687, "process": {"exists": false, "pid": 15687}, "restart_count": 112, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 113}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.53, "sample_count": 1161, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55250, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:31:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T18:31:02+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T18:31:02+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T18:31:02.832152+09:00", "mtime_epoch": 1782898262.8321517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:31:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.262, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.034, "pushed": 4.034, "read": 4.034}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.489, "count": 120, "max_ms": 11.217, "p95_ms": 10.366}, "gstreamer_push": {"avg_ms": 0.911, "count": 120, "max_ms": 2.901, "p95_ms": 0.957}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.706, "count": 120, "max_ms": 73.297, "p95_ms": 65.366}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.687, "count": 120, "max_ms": 91.137, "p95_ms": 85.552}, "yolo11_640_fire_11class": {"avg_ms": 54.2, "count": 120, "max_ms": 76.557, "p95_ms": 71.524}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.078, "count": 120, "max_ms": 92.101, "p95_ms": 88.871}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.962, "count": 120, "max_ms": 317.57, "p95_ms": 269.907}, "read_interval": {"avg_ms": 249.593, "count": 120, "max_ms": 328.215, "p95_ms": 282.602}, "triton_wait": {"avg_ms": 68.844, "count": 120, "max_ms": 92.113, "p95_ms": 89.513}, "visualize": {"avg_ms": 11.195, "count": 120, "max_ms": 15.943, "p95_ms": 12.258}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.749, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 460}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4004.559, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:31:02+09:00", "last_error_summary": "[h264 @ 0x5625467ef180] cabac decode of qscale diff failed at 49 44 [h264 @ 0x5625467ef180] error while decoding MB 49 44, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 98, "h264_decode_error_count": 101, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:31:02+09:00", "last_error_summary": "[h264 @ 0x5625467ef180] cabac decode of qscale diff failed at 49 44 [h264 @ 0x5625467ef180] error while decoding MB 49 44, bytestream -5", "last_exit_at": "2026-07-01T18:31:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.326, "last_started_at": "2026-07-01T18:29:12+09:00", "pid": 15687, "process": {"exists": false, "pid": 15687}, "restart_count": 112, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 113}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.53, "sample_count": 1161, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55251, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:31:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T18:31:03+09:00", "last_frame_read_age_sec": 0.256, "last_frame_read_at": "2026-07-01T18:31:02+09:00", "mjpeg": {"age_sec": 0.252, "exists": true, "mtime": "2026-07-01T18:31:02.832152+09:00", "mtime_epoch": 1782898262.8321517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:31:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.262, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.04, "pushed": 4.04, "read": 4.006}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.504, "count": 120, "max_ms": 11.217, "p95_ms": 10.375}, "gstreamer_push": {"avg_ms": 0.911, "count": 121, "max_ms": 2.901, "p95_ms": 0.957}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.656, "count": 120, "max_ms": 73.297, "p95_ms": 65.366}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.479, "count": 120, "max_ms": 91.137, "p95_ms": 84.834}, "yolo11_640_fire_11class": {"avg_ms": 54.151, "count": 120, "max_ms": 76.557, "p95_ms": 71.524}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.988, "count": 120, "max_ms": 92.101, "p95_ms": 88.871}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.05, "count": 121, "max_ms": 317.57, "p95_ms": 269.907}, "read_interval": {"avg_ms": 249.593, "count": 120, "max_ms": 328.215, "p95_ms": 282.602}, "triton_wait": {"avg_ms": 68.655, "count": 120, "max_ms": 92.113, "p95_ms": 89.513}, "visualize": {"avg_ms": 11.203, "count": 120, "max_ms": 15.943, "p95_ms": 12.258}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 460}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4004.551, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 98, "h264_decode_error_count": 101, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:31:02+09:00", "last_error_summary": "[h264 @ 0x5625467ef180] cabac decode of qscale diff failed at 49 44 [h264 @ 0x5625467ef180] error while decoding MB 49 44, bytestream -5", "last_exit_at": "2026-07-01T18:31:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.329, "last_started_at": "2026-07-01T18:29:12+09:00", "pid": 15687, "process": {"exists": false, "pid": 15687}, "restart_count": 112, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 113}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.53, "sample_count": 1161, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55251, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:31:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T18:31:03+09:00", "last_frame_read_age_sec": 0.259, "last_frame_read_at": "2026-07-01T18:31:02+09:00", "mjpeg": {"age_sec": 0.255, "exists": true, "mtime": "2026-07-01T18:31:02.832152+09:00", "mtime_epoch": 1782898262.8321517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:31:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.262, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.039, "pushed": 4.039, "read": 4.006}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.504, "count": 120, "max_ms": 11.217, "p95_ms": 10.375}, "gstreamer_push": {"avg_ms": 0.911, "count": 121, "max_ms": 2.901, "p95_ms": 0.957}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.656, "count": 120, "max_ms": 73.297, "p95_ms": 65.366}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.479, "count": 120, "max_ms": 91.137, "p95_ms": 84.834}, "yolo11_640_fire_11class": {"avg_ms": 54.151, "count": 120, "max_ms": 76.557, "p95_ms": 71.524}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.988, "count": 120, "max_ms": 92.101, "p95_ms": 88.871}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.05, "count": 121, "max_ms": 317.57, "p95_ms": 269.907}, "read_interval": {"avg_ms": 249.593, "count": 120, "max_ms": 328.215, "p95_ms": 282.602}, "triton_wait": {"avg_ms": 68.655, "count": 120, "max_ms": 92.113, "p95_ms": 89.513}, "visualize": {"avg_ms": 11.203, "count": 120, "max_ms": 15.943, "p95_ms": 12.258}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 460}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4128.941, "limit_mb": 8796093022207.996}, "detail": {"pid": 15794, "restart_count": 113, "start_count": 114}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 98, "h264_decode_error_count": 101, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:31:02+09:00", "last_error_summary": "[h264 @ 0x5625467ef180] cabac decode of qscale diff failed at 49 44 [h264 @ 0x5625467ef180] error while decoding MB 49 44, bytestream -5", "last_exit_at": "2026-07-01T18:31:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:31:05+09:00", "pid": 15794, "process": {"exists": true, "fd_count": 5, "hwm_mb": 150.035, "name": "ffmpeg", "pid": 15794, "rss_mb": 150.035, "state": "S (sleeping)", "threads": 106}, "restart_count": 113, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 114}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.804, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55251, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:31:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.65, "last_frame_pushed_at": "2026-07-01T18:31:03+09:00", "last_frame_read_age_sec": 4.902, "last_frame_read_at": "2026-07-01T18:31:02+09:00", "mjpeg": {"age_sec": 4.634, "exists": true, "mtime": "2026-07-01T18:31:03.096159+09:00", "mtime_epoch": 1782898263.0961587, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186519}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:31:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 885.262, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.4, "pushed": 3.4, "read": 3.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.533, "count": 101, "max_ms": 11.217, "p95_ms": 10.375}, "gstreamer_push": {"avg_ms": 0.92, "count": 102, "max_ms": 2.901, "p95_ms": 1.003}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.416, "count": 101, "max_ms": 73.297, "p95_ms": 66.73}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.957, "count": 101, "max_ms": 91.137, "p95_ms": 85.552}, "yolo11_640_fire_11class": {"avg_ms": 54.517, "count": 101, "max_ms": 76.557, "p95_ms": 71.616}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.582, "count": 101, "max_ms": 92.101, "p95_ms": 89.066}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.452, "count": 102, "max_ms": 317.57, "p95_ms": 273.804}, "read_interval": {"avg_ms": 251.01, "count": 101, "max_ms": 328.215, "p95_ms": 283.016}, "triton_wait": {"avg_ms": 69.192, "count": 101, "max_ms": 92.113, "p95_ms": 89.614}, "visualize": {"avg_ms": 11.207, "count": 101, "max_ms": 15.943, "p95_ms": 12.203}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 464}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4014.117, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f6110309c0] error while decoding MB 52 30, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 99, "h264_decode_error_count": 102, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:32:59+09:00", "last_error_summary": "[h264 @ 0x55f6110309c0] error while decoding MB 52 30, bytestream -9", "last_exit_at": "2026-07-01T18:31:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.614, "last_started_at": "2026-07-01T18:31:05+09:00", "pid": 15794, "process": {"exists": false, "pid": 15794}, "restart_count": 113, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 114}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.496, "sample_count": 1172, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55701, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:32:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T18:32:59+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T18:32:59+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T18:32:59.299217+09:00", "mtime_epoch": 1782898379.299217, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184185}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:32:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 888.109, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.969, "pushed": 3.969, "read": 3.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.564, "count": 118, "max_ms": 13.133, "p95_ms": 11.18}, "gstreamer_push": {"avg_ms": 0.945, "count": 119, "max_ms": 2.961, "p95_ms": 1.806}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.563, "count": 118, "max_ms": 72.233, "p95_ms": 62.884}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.046, "count": 118, "max_ms": 92.192, "p95_ms": 90.142}, "yolo11_640_fire_11class": {"avg_ms": 55.251, "count": 118, "max_ms": 74.629, "p95_ms": 72.22}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.958, "count": 118, "max_ms": 93.283, "p95_ms": 90.795}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.847, "count": 119, "max_ms": 343.356, "p95_ms": 279.082}, "read_interval": {"avg_ms": 253.518, "count": 119, "max_ms": 353.466, "p95_ms": 288.064}, "triton_wait": {"avg_ms": 70.832, "count": 118, "max_ms": 93.295, "p95_ms": 91.428}, "visualize": {"avg_ms": 11.317, "count": 118, "max_ms": 16.737, "p95_ms": 12.799}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 464}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4013.227, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:32:59+09:00", "last_error_summary": "[h264 @ 0x55f6110309c0] error while decoding MB 52 30, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 99, "h264_decode_error_count": 102, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:32:59+09:00", "last_error_summary": "[h264 @ 0x55f6110309c0] error while decoding MB 52 30, bytestream -9", "last_exit_at": "2026-07-01T18:32:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.792, "last_started_at": "2026-07-01T18:31:05+09:00", "pid": 15794, "process": {"exists": false, "pid": 15794}, "restart_count": 113, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 114}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.496, "sample_count": 1172, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55702, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:32:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:32:59+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T18:32:59+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T18:32:59.299217+09:00", "mtime_epoch": 1782898379.299217, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184185}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:32:59+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 888.109, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.978, "pushed": 3.978, "read": 3.944}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.563, "count": 118, "max_ms": 13.133, "p95_ms": 11.18}, "gstreamer_push": {"avg_ms": 0.945, "count": 119, "max_ms": 2.961, "p95_ms": 1.806}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.668, "count": 118, "max_ms": 72.233, "p95_ms": 62.884}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.126, "count": 118, "max_ms": 92.192, "p95_ms": 90.142}, "yolo11_640_fire_11class": {"avg_ms": 55.342, "count": 118, "max_ms": 74.629, "p95_ms": 72.22}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.051, "count": 118, "max_ms": 93.283, "p95_ms": 90.795}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.76, "count": 119, "max_ms": 343.356, "p95_ms": 279.082}, "read_interval": {"avg_ms": 253.579, "count": 118, "max_ms": 353.466, "p95_ms": 288.064}, "triton_wait": {"avg_ms": 70.918, "count": 118, "max_ms": 93.295, "p95_ms": 91.428}, "visualize": {"avg_ms": 11.284, "count": 118, "max_ms": 16.737, "p95_ms": 12.753}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 464}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4013.344, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 99, "h264_decode_error_count": 102, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:32:59+09:00", "last_error_summary": "[h264 @ 0x55f6110309c0] error while decoding MB 52 30, bytestream -9", "last_exit_at": "2026-07-01T18:32:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.795, "last_started_at": "2026-07-01T18:31:05+09:00", "pid": 15794, "process": {"exists": false, "pid": 15794}, "restart_count": 113, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 114}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.496, "sample_count": 1172, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55702, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:32:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:32:59+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T18:32:59+09:00", "mjpeg": {"age_sec": 0.225, "exists": true, "mtime": "2026-07-01T18:32:59.299217+09:00", "mtime_epoch": 1782898379.299217, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184185}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:32:59+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 888.109, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.977, "pushed": 3.977, "read": 3.944}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.563, "count": 118, "max_ms": 13.133, "p95_ms": 11.18}, "gstreamer_push": {"avg_ms": 0.945, "count": 119, "max_ms": 2.961, "p95_ms": 1.806}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.668, "count": 118, "max_ms": 72.233, "p95_ms": 62.884}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.126, "count": 118, "max_ms": 92.192, "p95_ms": 90.142}, "yolo11_640_fire_11class": {"avg_ms": 55.342, "count": 118, "max_ms": 74.629, "p95_ms": 72.22}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.051, "count": 118, "max_ms": 93.283, "p95_ms": 90.795}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.76, "count": 119, "max_ms": 343.356, "p95_ms": 279.082}, "read_interval": {"avg_ms": 253.579, "count": 118, "max_ms": 353.466, "p95_ms": 288.064}, "triton_wait": {"avg_ms": 70.918, "count": 118, "max_ms": 93.295, "p95_ms": 91.428}, "visualize": {"avg_ms": 11.284, "count": 118, "max_ms": 16.737, "p95_ms": 12.753}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 464}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4123.316, "limit_mb": 8796093022207.996}, "detail": {"pid": 15901, "restart_count": 114, "start_count": 115}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 99, "h264_decode_error_count": 102, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:32:59+09:00", "last_error_summary": "[h264 @ 0x55f6110309c0] error while decoding MB 52 30, bytestream -9", "last_exit_at": "2026-07-01T18:33:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:33:02+09:00", "pid": 15901, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.742, "name": "ffmpeg", "pid": 15901, "rss_mb": 136.742, "state": "S (sleeping)", "threads": 106}, "restart_count": 114, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 115}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.46, "sample_count": 1153, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55702, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:32:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.61, "last_frame_pushed_at": "2026-07-01T18:32:59+09:00", "last_frame_read_age_sec": 4.838, "last_frame_read_at": "2026-07-01T18:32:59+09:00", "mjpeg": {"age_sec": 4.593, "exists": true, "mtime": "2026-07-01T18:32:59.535223+09:00", "mtime_epoch": 1782898379.535223, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182523}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:33:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 888.109, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.358, "pushed": 3.358, "read": 3.324}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.592, "count": 100, "max_ms": 13.133, "p95_ms": 10.774}, "gstreamer_push": {"avg_ms": 0.948, "count": 100, "max_ms": 2.961, "p95_ms": 1.802}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.013, "count": 100, "max_ms": 72.233, "p95_ms": 62.884}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.348, "count": 100, "max_ms": 92.192, "p95_ms": 90.142}, "yolo11_640_fire_11class": {"avg_ms": 55.895, "count": 100, "max_ms": 74.629, "p95_ms": 72.22}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.6, "count": 100, "max_ms": 93.283, "p95_ms": 90.795}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.605, "count": 100, "max_ms": 343.356, "p95_ms": 279.082}, "read_interval": {"avg_ms": 254.454, "count": 99, "max_ms": 353.466, "p95_ms": 318.144}, "triton_wait": {"avg_ms": 71.273, "count": 100, "max_ms": 93.295, "p95_ms": 91.428}, "visualize": {"avg_ms": 11.231, "count": 100, "max_ms": 16.737, "p95_ms": 12.236}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.781, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4009.461, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c2c94fc680] error while decoding MB 14 35, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 100, "h264_decode_error_count": 103, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:34:51+09:00", "last_error_summary": "[h264 @ 0x55c2c94fc680] error while decoding MB 14 35, bytestream -5", "last_exit_at": "2026-07-01T18:33:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.171, "last_started_at": "2026-07-01T18:33:02+09:00", "pid": 15901, "process": {"exists": false, "pid": 15901}, "restart_count": 114, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 115}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.496, "sample_count": 1178, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56141, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:34:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T18:34:51+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T18:34:51+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T18:34:51.254001+09:00", "mtime_epoch": 1782898491.2540011, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188412}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:34:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.199, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.131, "pushed": 4.131, "read": 4.131}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.473, "count": 123, "max_ms": 14.738, "p95_ms": 10.874}, "gstreamer_push": {"avg_ms": 0.893, "count": 123, "max_ms": 1.805, "p95_ms": 1.017}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.648, "count": 123, "max_ms": 76.186, "p95_ms": 61.307}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.383, "count": 123, "max_ms": 91.763, "p95_ms": 84.546}, "yolo11_640_fire_11class": {"avg_ms": 51.771, "count": 123, "max_ms": 74.936, "p95_ms": 69.944}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.058, "count": 123, "max_ms": 89.123, "p95_ms": 86.558}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.942, "count": 123, "max_ms": 370.679, "p95_ms": 253.731}, "read_interval": {"avg_ms": 243.622, "count": 123, "max_ms": 381.437, "p95_ms": 262.948}, "triton_wait": {"avg_ms": 66.028, "count": 123, "max_ms": 91.775, "p95_ms": 86.75}, "visualize": {"avg_ms": 11.033, "count": 123, "max_ms": 12.713, "p95_ms": 12.316}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.772, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4008.574, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:34:51+09:00", "last_error_summary": "[h264 @ 0x55c2c94fc680] error while decoding MB 14 35, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 100, "h264_decode_error_count": 103, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:34:51+09:00", "last_error_summary": "[h264 @ 0x55c2c94fc680] error while decoding MB 14 35, bytestream -5", "last_exit_at": "2026-07-01T18:34:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.375, "last_started_at": "2026-07-01T18:33:02+09:00", "pid": 15901, "process": {"exists": false, "pid": 15901}, "restart_count": 114, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 115}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.496, "sample_count": 1178, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56142, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:34:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:34:51+09:00", "last_frame_read_age_sec": 0.257, "last_frame_read_at": "2026-07-01T18:34:51+09:00", "mjpeg": {"age_sec": 0.248, "exists": true, "mtime": "2026-07-01T18:34:51.254001+09:00", "mtime_epoch": 1782898491.2540011, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188412}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:34:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.199, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.137, "pushed": 4.137, "read": 4.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.512, "count": 123, "max_ms": 14.997, "p95_ms": 10.967}, "gstreamer_push": {"avg_ms": 0.892, "count": 124, "max_ms": 1.805, "p95_ms": 1.017}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.66, "count": 123, "max_ms": 76.186, "p95_ms": 61.307}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.36, "count": 123, "max_ms": 91.763, "p95_ms": 84.546}, "yolo11_640_fire_11class": {"avg_ms": 51.796, "count": 123, "max_ms": 74.936, "p95_ms": 69.944}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.023, "count": 123, "max_ms": 89.123, "p95_ms": 86.558}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.095, "count": 124, "max_ms": 370.679, "p95_ms": 253.731}, "read_interval": {"avg_ms": 243.622, "count": 123, "max_ms": 381.437, "p95_ms": 262.948}, "triton_wait": {"avg_ms": 65.999, "count": 123, "max_ms": 91.775, "p95_ms": 86.75}, "visualize": {"avg_ms": 11.04, "count": 123, "max_ms": 12.713, "p95_ms": 12.345}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4008.566, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 100, "h264_decode_error_count": 103, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:34:51+09:00", "last_error_summary": "[h264 @ 0x55c2c94fc680] error while decoding MB 14 35, bytestream -5", "last_exit_at": "2026-07-01T18:34:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.378, "last_started_at": "2026-07-01T18:33:02+09:00", "pid": 15901, "process": {"exists": false, "pid": 15901}, "restart_count": 114, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 115}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.496, "sample_count": 1178, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56142, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:34:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:34:51+09:00", "last_frame_read_age_sec": 0.26, "last_frame_read_at": "2026-07-01T18:34:51+09:00", "mjpeg": {"age_sec": 0.251, "exists": true, "mtime": "2026-07-01T18:34:51.254001+09:00", "mtime_epoch": 1782898491.2540011, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188412}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:34:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.199, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.136, "pushed": 4.136, "read": 4.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.512, "count": 123, "max_ms": 14.997, "p95_ms": 10.967}, "gstreamer_push": {"avg_ms": 0.892, "count": 124, "max_ms": 1.805, "p95_ms": 1.017}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.66, "count": 123, "max_ms": 76.186, "p95_ms": 61.307}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.36, "count": 123, "max_ms": 91.763, "p95_ms": 84.546}, "yolo11_640_fire_11class": {"avg_ms": 51.796, "count": 123, "max_ms": 74.936, "p95_ms": 69.944}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.023, "count": 123, "max_ms": 89.123, "p95_ms": 86.558}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.095, "count": 124, "max_ms": 370.679, "p95_ms": 253.731}, "read_interval": {"avg_ms": 243.622, "count": 123, "max_ms": 381.437, "p95_ms": 262.948}, "triton_wait": {"avg_ms": 65.999, "count": 123, "max_ms": 91.775, "p95_ms": 86.75}, "visualize": {"avg_ms": 11.04, "count": 123, "max_ms": 12.713, "p95_ms": 12.345}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 467}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4118.871, "limit_mb": 8796093022207.996}, "detail": {"pid": 16008, "restart_count": 115, "start_count": 116}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 100, "h264_decode_error_count": 103, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:34:51+09:00", "last_error_summary": "[h264 @ 0x55c2c94fc680] error while decoding MB 14 35, bytestream -5", "last_exit_at": "2026-07-01T18:34:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:34:54+09:00", "pid": 16008, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.176, "name": "ffmpeg", "pid": 16008, "rss_mb": 139.176, "state": "S (sleeping)", "threads": 106}, "restart_count": 115, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 116}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.797, "sample_count": 1159, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56142, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:34:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.697, "last_frame_pushed_at": "2026-07-01T18:34:51+09:00", "last_frame_read_age_sec": 4.951, "last_frame_read_at": "2026-07-01T18:34:51+09:00", "mjpeg": {"age_sec": 4.678, "exists": true, "mtime": "2026-07-01T18:34:51.518008+09:00", "mtime_epoch": 1782898491.5180075, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188700}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:34:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 883.199, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.522, "pushed": 3.522, "read": 3.489}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.338, "count": 105, "max_ms": 14.997, "p95_ms": 10.874}, "gstreamer_push": {"avg_ms": 0.895, "count": 105, "max_ms": 1.805, "p95_ms": 1.017}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.56, "count": 105, "max_ms": 76.186, "p95_ms": 61.471}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.071, "count": 105, "max_ms": 91.763, "p95_ms": 84.546}, "yolo11_640_fire_11class": {"avg_ms": 52.356, "count": 105, "max_ms": 74.794, "p95_ms": 69.944}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.534, "count": 105, "max_ms": 89.123, "p95_ms": 86.558}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.663, "count": 105, "max_ms": 287.799, "p95_ms": 252.913}, "read_interval": {"avg_ms": 241.152, "count": 104, "max_ms": 296.786, "p95_ms": 261.721}, "triton_wait": {"avg_ms": 66.608, "count": 105, "max_ms": 91.775, "p95_ms": 86.75}, "visualize": {"avg_ms": 10.943, "count": 105, "max_ms": 12.713, "p95_ms": 12.345}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.809, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 471}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4018.84, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:34:51+09:00", "last_error_summary": "[h264 @ 0x55c2c94fc680] error while decoding MB 14 35, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 100, "h264_decode_error_count": 103, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:34:51+09:00", "last_error_summary": "[h264 @ 0x55c2c94fc680] error while decoding MB 14 35, bytestream -5", "last_exit_at": "2026-07-01T18:36:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.781, "last_started_at": "2026-07-01T18:34:54+09:00", "pid": 16008, "process": {"exists": false, "pid": 16008}, "restart_count": 115, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 116}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.184, "sample_count": 1183, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56581, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:36:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:36:43+09:00", "last_frame_read_age_sec": 0.227, "last_frame_read_at": "2026-07-01T18:36:43+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-01T18:36:43.756671+09:00", "mtime_epoch": 1782898603.7566707, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189511}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:36:43+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 888.5, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.134, "pushed": 4.134, "read": 4.101}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.207, "count": 123, "max_ms": 13.127, "p95_ms": 10.095}, "gstreamer_push": {"avg_ms": 0.894, "count": 124, "max_ms": 1.803, "p95_ms": 1.032}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.876, "count": 123, "max_ms": 70.599, "p95_ms": 59.254}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.422, "count": 123, "max_ms": 93.299, "p95_ms": 83.434}, "yolo11_640_fire_11class": {"avg_ms": 52.758, "count": 123, "max_ms": 71.421, "p95_ms": 67.856}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.453, "count": 123, "max_ms": 94.268, "p95_ms": 81.749}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.368, "count": 124, "max_ms": 291.749, "p95_ms": 259.145}, "read_interval": {"avg_ms": 243.932, "count": 123, "max_ms": 302.071, "p95_ms": 268.657}, "triton_wait": {"avg_ms": 67.132, "count": 123, "max_ms": 94.279, "p95_ms": 83.563}, "visualize": {"avg_ms": 11.098, "count": 123, "max_ms": 14.462, "p95_ms": 12.509}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 471}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4018.832, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 100, "h264_decode_error_count": 103, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:34:51+09:00", "last_error_summary": "[h264 @ 0x55c2c94fc680] error while decoding MB 14 35, bytestream -5", "last_exit_at": "2026-07-01T18:36:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.785, "last_started_at": "2026-07-01T18:34:54+09:00", "pid": 16008, "process": {"exists": false, "pid": 16008}, "restart_count": 115, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 116}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.184, "sample_count": 1183, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56581, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:36:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:36:43+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T18:36:43+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T18:36:43.756671+09:00", "mtime_epoch": 1782898603.7566707, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189511}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:36:43+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 888.5, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.134, "pushed": 4.134, "read": 4.1}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.207, "count": 123, "max_ms": 13.127, "p95_ms": 10.095}, "gstreamer_push": {"avg_ms": 0.894, "count": 124, "max_ms": 1.803, "p95_ms": 1.032}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.876, "count": 123, "max_ms": 70.599, "p95_ms": 59.254}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.422, "count": 123, "max_ms": 93.299, "p95_ms": 83.434}, "yolo11_640_fire_11class": {"avg_ms": 52.758, "count": 123, "max_ms": 71.421, "p95_ms": 67.856}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.453, "count": 123, "max_ms": 94.268, "p95_ms": 81.749}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.368, "count": 124, "max_ms": 291.749, "p95_ms": 259.145}, "read_interval": {"avg_ms": 243.932, "count": 123, "max_ms": 302.071, "p95_ms": 268.657}, "triton_wait": {"avg_ms": 67.132, "count": 123, "max_ms": 94.279, "p95_ms": 83.563}, "visualize": {"avg_ms": 11.098, "count": 123, "max_ms": 14.462, "p95_ms": 12.509}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 471}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4124.113, "limit_mb": 8796093022207.996}, "detail": {"pid": 16115, "restart_count": 116, "start_count": 117}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 100, "h264_decode_error_count": 103, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:34:51+09:00", "last_error_summary": "[h264 @ 0x55c2c94fc680] error while decoding MB 14 35, bytestream -5", "last_exit_at": "2026-07-01T18:36:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:36:46+09:00", "pid": 16115, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.289, "name": "ffmpeg", "pid": 16115, "rss_mb": 139.289, "state": "S (sleeping)", "threads": 106}, "restart_count": 116, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 117}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.184, "sample_count": 1163, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 56581, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:36:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.959, "last_frame_pushed_at": "2026-07-01T18:36:43+09:00", "last_frame_read_age_sec": 5.183, "last_frame_read_at": "2026-07-01T18:36:43+09:00", "mjpeg": {"age_sec": 4.945, "exists": true, "mtime": "2026-07-01T18:36:43.988676+09:00", "mtime_epoch": 1782898603.988676, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189673}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:36:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.047, "name": "celery", "pid": 237, "rss_mb": 888.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.447, "pushed": 3.447, "read": 3.414}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.222, "count": 102, "max_ms": 13.127, "p95_ms": 10.095}, "gstreamer_push": {"avg_ms": 0.887, "count": 103, "max_ms": 1.754, "p95_ms": 1.032}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.964, "count": 102, "max_ms": 70.599, "p95_ms": 58.883}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.547, "count": 102, "max_ms": 93.299, "p95_ms": 83.434}, "yolo11_640_fire_11class": {"avg_ms": 52.631, "count": 102, "max_ms": 71.421, "p95_ms": 65.673}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.564, "count": 102, "max_ms": 94.268, "p95_ms": 81.749}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.808, "count": 103, "max_ms": 291.749, "p95_ms": 272.721}, "read_interval": {"avg_ms": 244.349, "count": 102, "max_ms": 302.071, "p95_ms": 282.243}, "triton_wait": {"avg_ms": 67.167, "count": 102, "max_ms": 94.279, "p95_ms": 83.446}, "visualize": {"avg_ms": 11.103, "count": 102, "max_ms": 14.462, "p95_ms": 12.509}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 475}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4020.578, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556049478e40] error while decoding MB 6 32, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 101, "h264_decode_error_count": 104, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:38:36+09:00", "last_error_summary": "[h264 @ 0x556049478e40] error while decoding MB 6 32, bytestream -5", "last_exit_at": "2026-07-01T18:36:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.317, "last_started_at": "2026-07-01T18:36:46+09:00", "pid": 16115, "process": {"exists": false, "pid": 16115}, "restart_count": 116, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 117}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.33, "sample_count": 1182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57015, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:38:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-01T18:38:36+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-01T18:38:36+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-01T18:38:36.199236+09:00", "mtime_epoch": 1782898716.1992357, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:38:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 888.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.094, "pushed": 4.094, "read": 4.094}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.405, "count": 122, "max_ms": 12.514, "p95_ms": 10.25}, "gstreamer_push": {"avg_ms": 0.924, "count": 122, "max_ms": 2.821, "p95_ms": 1.147}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.518, "count": 122, "max_ms": 80.662, "p95_ms": 63.127}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.651, "count": 122, "max_ms": 97.677, "p95_ms": 88.758}, "yolo11_640_fire_11class": {"avg_ms": 54.797, "count": 122, "max_ms": 84.346, "p95_ms": 72.706}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.89, "count": 122, "max_ms": 101.298, "p95_ms": 89.001}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.25, "count": 122, "max_ms": 321.161, "p95_ms": 275.081}, "read_interval": {"avg_ms": 245.774, "count": 122, "max_ms": 330.205, "p95_ms": 284.697}, "triton_wait": {"avg_ms": 69.478, "count": 122, "max_ms": 101.31, "p95_ms": 89.012}, "visualize": {"avg_ms": 10.961, "count": 122, "max_ms": 14.837, "p95_ms": 12.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.799, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 475}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4019.418, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:38:36+09:00", "last_error_summary": "[h264 @ 0x556049478e40] error while decoding MB 6 32, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 101, "h264_decode_error_count": 104, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:38:36+09:00", "last_error_summary": "[h264 @ 0x556049478e40] error while decoding MB 6 32, bytestream -5", "last_exit_at": "2026-07-01T18:38:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.529, "last_started_at": "2026-07-01T18:36:46+09:00", "pid": 16115, "process": {"exists": false, "pid": 16115}, "restart_count": 116, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 117}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.33, "sample_count": 1182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57016, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:38:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:38:36+09:00", "last_frame_read_age_sec": 0.272, "last_frame_read_at": "2026-07-01T18:38:36+09:00", "mjpeg": {"age_sec": 0.262, "exists": true, "mtime": "2026-07-01T18:38:36.199236+09:00", "mtime_epoch": 1782898716.1992357, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:38:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 888.633, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.1, "pushed": 4.1, "read": 4.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.405, "count": 122, "max_ms": 12.514, "p95_ms": 10.25}, "gstreamer_push": {"avg_ms": 0.924, "count": 122, "max_ms": 2.821, "p95_ms": 1.147}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.461, "count": 122, "max_ms": 80.662, "p95_ms": 63.127}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.807, "count": 122, "max_ms": 97.677, "p95_ms": 88.758}, "yolo11_640_fire_11class": {"avg_ms": 54.833, "count": 122, "max_ms": 84.346, "p95_ms": 72.706}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.95, "count": 122, "max_ms": 101.298, "p95_ms": 89.001}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.43, "count": 122, "max_ms": 321.161, "p95_ms": 275.081}, "read_interval": {"avg_ms": 245.698, "count": 121, "max_ms": 330.205, "p95_ms": 284.697}, "triton_wait": {"avg_ms": 69.617, "count": 122, "max_ms": 101.31, "p95_ms": 89.012}, "visualize": {"avg_ms": 10.956, "count": 122, "max_ms": 14.837, "p95_ms": 12.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.754, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 475}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4019.461, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 101, "h264_decode_error_count": 104, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:38:36+09:00", "last_error_summary": "[h264 @ 0x556049478e40] error while decoding MB 6 32, bytestream -5", "last_exit_at": "2026-07-01T18:38:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.533, "last_started_at": "2026-07-01T18:36:46+09:00", "pid": 16115, "process": {"exists": false, "pid": 16115}, "restart_count": 116, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 117}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.33, "sample_count": 1182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57016, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:38:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:38:36+09:00", "last_frame_read_age_sec": 0.275, "last_frame_read_at": "2026-07-01T18:38:36+09:00", "mjpeg": {"age_sec": 0.265, "exists": true, "mtime": "2026-07-01T18:38:36.199236+09:00", "mtime_epoch": 1782898716.1992357, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:38:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 888.633, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.1, "pushed": 4.1, "read": 4.066}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.405, "count": 122, "max_ms": 12.514, "p95_ms": 10.25}, "gstreamer_push": {"avg_ms": 0.924, "count": 122, "max_ms": 2.821, "p95_ms": 1.147}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.461, "count": 122, "max_ms": 80.662, "p95_ms": 63.127}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.807, "count": 122, "max_ms": 97.677, "p95_ms": 88.758}, "yolo11_640_fire_11class": {"avg_ms": 54.833, "count": 122, "max_ms": 84.346, "p95_ms": 72.706}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.95, "count": 122, "max_ms": 101.298, "p95_ms": 89.001}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.43, "count": 122, "max_ms": 321.161, "p95_ms": 275.081}, "read_interval": {"avg_ms": 245.698, "count": 121, "max_ms": 330.205, "p95_ms": 284.697}, "triton_wait": {"avg_ms": 69.617, "count": 122, "max_ms": 101.31, "p95_ms": 89.012}, "visualize": {"avg_ms": 10.956, "count": 122, "max_ms": 14.837, "p95_ms": 12.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.757, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 475}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4124.504, "limit_mb": 8796093022207.996}, "detail": {"pid": 16222, "restart_count": 117, "start_count": 118}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 101, "h264_decode_error_count": 104, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:38:36+09:00", "last_error_summary": "[h264 @ 0x556049478e40] error while decoding MB 6 32, bytestream -5", "last_exit_at": "2026-07-01T18:38:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:38:38+09:00", "pid": 16222, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.23, "name": "ffmpeg", "pid": 16222, "rss_mb": 139.23, "state": "S (sleeping)", "threads": 106}, "restart_count": 117, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 118}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.33, "sample_count": 1164, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57016, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:38:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.392, "last_frame_pushed_at": "2026-07-01T18:38:36+09:00", "last_frame_read_age_sec": 4.661, "last_frame_read_at": "2026-07-01T18:38:36+09:00", "mjpeg": {"age_sec": 4.375, "exists": true, "mtime": "2026-07-01T18:38:36.475242+09:00", "mtime_epoch": 1782898716.475242, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190806}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:38:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 888.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.523, "pushed": 3.523, "read": 3.49}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.442, "count": 105, "max_ms": 12.514, "p95_ms": 10.271}, "gstreamer_push": {"avg_ms": 0.918, "count": 105, "max_ms": 2.737, "p95_ms": 1.147}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.368, "count": 105, "max_ms": 80.662, "p95_ms": 62.774}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.931, "count": 105, "max_ms": 97.677, "p95_ms": 87.357}, "yolo11_640_fire_11class": {"avg_ms": 53.984, "count": 105, "max_ms": 84.346, "p95_ms": 70.073}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.171, "count": 105, "max_ms": 101.298, "p95_ms": 87.37}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.64, "count": 105, "max_ms": 321.161, "p95_ms": 267.956}, "read_interval": {"avg_ms": 243.829, "count": 104, "max_ms": 330.205, "p95_ms": 275.835}, "triton_wait": {"avg_ms": 68.719, "count": 105, "max_ms": 101.31, "p95_ms": 87.38}, "visualize": {"avg_ms": 10.917, "count": 105, "max_ms": 14.837, "p95_ms": 11.943}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.8, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 478}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4018.402, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559baf5cd680] error while decoding MB 53 20, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 102, "h264_decode_error_count": 105, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:40:29+09:00", "last_error_summary": "[h264 @ 0x559baf5cd680] error while decoding MB 53 20, bytestream -5", "last_exit_at": "2026-07-01T18:38:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.241, "last_started_at": "2026-07-01T18:38:38+09:00", "pid": 16222, "process": {"exists": false, "pid": 16222}, "restart_count": 117, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 118}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.255, "sample_count": 1167, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57442, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:40:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T18:40:29+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T18:40:29+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T18:40:29.045727+09:00", "mtime_epoch": 1782898829.0457268, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185338}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:40:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 883.887, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.872, "pushed": 3.872, "read": 3.872}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.933, "count": 115, "max_ms": 11.652, "p95_ms": 10.834}, "gstreamer_push": {"avg_ms": 0.955, "count": 116, "max_ms": 2.965, "p95_ms": 1.792}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.418, "count": 115, "max_ms": 72.64, "p95_ms": 69.48}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.506, "count": 115, "max_ms": 94.798, "p95_ms": 90.262}, "yolo11_640_fire_11class": {"avg_ms": 54.865, "count": 115, "max_ms": 74.094, "p95_ms": 71.16}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.158, "count": 115, "max_ms": 95.215, "p95_ms": 91.497}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.379, "count": 116, "max_ms": 339.867, "p95_ms": 275.824}, "read_interval": {"avg_ms": 259.978, "count": 116, "max_ms": 350.998, "p95_ms": 284.851}, "triton_wait": {"avg_ms": 70.954, "count": 115, "max_ms": 95.228, "p95_ms": 91.92}, "visualize": {"avg_ms": 11.563, "count": 115, "max_ms": 14.671, "p95_ms": 12.445}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 478}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4017.559, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:40:29+09:00", "last_error_summary": "[h264 @ 0x559baf5cd680] error while decoding MB 53 20, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 102, "h264_decode_error_count": 105, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:40:29+09:00", "last_error_summary": "[h264 @ 0x559baf5cd680] error while decoding MB 53 20, bytestream -5", "last_exit_at": "2026-07-01T18:40:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.419, "last_started_at": "2026-07-01T18:38:38+09:00", "pid": 16222, "process": {"exists": false, "pid": 16222}, "restart_count": 117, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 118}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.255, "sample_count": 1167, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57443, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:40:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T18:40:29+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T18:40:29+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T18:40:29.045727+09:00", "mtime_epoch": 1782898829.0457268, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185338}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:40:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 883.887, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.878, "pushed": 3.878, "read": 3.845}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.937, "count": 115, "max_ms": 11.652, "p95_ms": 10.834}, "gstreamer_push": {"avg_ms": 0.955, "count": 116, "max_ms": 2.965, "p95_ms": 1.792}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.386, "count": 115, "max_ms": 72.64, "p95_ms": 69.48}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.542, "count": 115, "max_ms": 94.798, "p95_ms": 90.262}, "yolo11_640_fire_11class": {"avg_ms": 54.891, "count": 115, "max_ms": 74.094, "p95_ms": 71.16}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.161, "count": 115, "max_ms": 95.215, "p95_ms": 91.497}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.161, "count": 116, "max_ms": 339.867, "p95_ms": 275.824}, "read_interval": {"avg_ms": 259.986, "count": 115, "max_ms": 350.998, "p95_ms": 284.851}, "triton_wait": {"avg_ms": 70.964, "count": 115, "max_ms": 95.228, "p95_ms": 91.92}, "visualize": {"avg_ms": 11.568, "count": 115, "max_ms": 14.671, "p95_ms": 12.445}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 478}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4017.527, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 102, "h264_decode_error_count": 105, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:40:29+09:00", "last_error_summary": "[h264 @ 0x559baf5cd680] error while decoding MB 53 20, bytestream -5", "last_exit_at": "2026-07-01T18:40:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.422, "last_started_at": "2026-07-01T18:38:38+09:00", "pid": 16222, "process": {"exists": false, "pid": 16222}, "restart_count": 117, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 118}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.255, "sample_count": 1167, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57443, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:40:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T18:40:29+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-01T18:40:29+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-01T18:40:29.045727+09:00", "mtime_epoch": 1782898829.0457268, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185338}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:40:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 883.887, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.878, "pushed": 3.878, "read": 3.844}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.937, "count": 115, "max_ms": 11.652, "p95_ms": 10.834}, "gstreamer_push": {"avg_ms": 0.955, "count": 116, "max_ms": 2.965, "p95_ms": 1.792}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.386, "count": 115, "max_ms": 72.64, "p95_ms": 69.48}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.542, "count": 115, "max_ms": 94.798, "p95_ms": 90.262}, "yolo11_640_fire_11class": {"avg_ms": 54.891, "count": 115, "max_ms": 74.094, "p95_ms": 71.16}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.161, "count": 115, "max_ms": 95.215, "p95_ms": 91.497}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.161, "count": 116, "max_ms": 339.867, "p95_ms": 275.824}, "read_interval": {"avg_ms": 259.986, "count": 115, "max_ms": 350.998, "p95_ms": 284.851}, "triton_wait": {"avg_ms": 70.964, "count": 115, "max_ms": 95.228, "p95_ms": 91.92}, "visualize": {"avg_ms": 11.568, "count": 115, "max_ms": 14.671, "p95_ms": 12.445}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 478}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4133.176, "limit_mb": 8796093022207.996}, "detail": {"pid": 16329, "restart_count": 118, "start_count": 119}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 102, "h264_decode_error_count": 105, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:40:29+09:00", "last_error_summary": "[h264 @ 0x559baf5cd680] error while decoding MB 53 20, bytestream -5", "last_exit_at": "2026-07-01T18:40:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:40:31+09:00", "pid": 16329, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.793, "name": "ffmpeg", "pid": 16329, "rss_mb": 147.793, "state": "S (sleeping)", "threads": 106}, "restart_count": 118, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 119}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.063, "sample_count": 1150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57443, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:40:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.452, "last_frame_pushed_at": "2026-07-01T18:40:29+09:00", "last_frame_read_age_sec": 4.678, "last_frame_read_at": "2026-07-01T18:40:29+09:00", "mjpeg": {"age_sec": 4.435, "exists": true, "mtime": "2026-07-01T18:40:29.281732+09:00", "mtime_epoch": 1782898829.2817318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184098}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:40:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 883.887, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.289, "pushed": 3.289, "read": 3.256}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.896, "count": 98, "max_ms": 11.204, "p95_ms": 10.814}, "gstreamer_push": {"avg_ms": 0.963, "count": 98, "max_ms": 2.965, "p95_ms": 1.802}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.063, "count": 98, "max_ms": 72.64, "p95_ms": 69.57}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.804, "count": 98, "max_ms": 94.798, "p95_ms": 90.718}, "yolo11_640_fire_11class": {"avg_ms": 55.653, "count": 98, "max_ms": 74.094, "p95_ms": 71.498}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.27, "count": 98, "max_ms": 95.215, "p95_ms": 91.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 251.524, "count": 98, "max_ms": 339.867, "p95_ms": 275.824}, "read_interval": {"avg_ms": 261.413, "count": 97, "max_ms": 350.998, "p95_ms": 284.851}, "triton_wait": {"avg_ms": 72.202, "count": 98, "max_ms": 95.228, "p95_ms": 91.93}, "visualize": {"avg_ms": 11.577, "count": 98, "max_ms": 14.671, "p95_ms": 12.473}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.793, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4019.281, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556e59640680] error while decoding MB 41 19, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 103, "h264_decode_error_count": 106, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:42:21+09:00", "last_error_summary": "[h264 @ 0x556e59640680] error while decoding MB 41 19, bytestream -5", "last_exit_at": "2026-07-01T18:40:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.827, "last_started_at": "2026-07-01T18:40:31+09:00", "pid": 16329, "process": {"exists": false, "pid": 16329}, "restart_count": 118, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 119}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.452, "sample_count": 1148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57862, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:42:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T18:42:21+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T18:42:21+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T18:42:21.496143+09:00", "mtime_epoch": 1782898941.4961426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192081}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:42:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 886.445, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.946, "pushed": 3.946, "read": 3.946}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.059, "count": 117, "max_ms": 13.402, "p95_ms": 12.258}, "gstreamer_push": {"avg_ms": 0.942, "count": 118, "max_ms": 2.889, "p95_ms": 1.768}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.78, "count": 117, "max_ms": 69.723, "p95_ms": 62.845}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.225, "count": 117, "max_ms": 89.737, "p95_ms": 86.327}, "yolo11_640_fire_11class": {"avg_ms": 53.032, "count": 117, "max_ms": 71.246, "p95_ms": 68.958}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.757, "count": 117, "max_ms": 90.635, "p95_ms": 88.426}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.225, "count": 118, "max_ms": 322.091, "p95_ms": 273.789}, "read_interval": {"avg_ms": 254.853, "count": 118, "max_ms": 332.875, "p95_ms": 282.77}, "triton_wait": {"avg_ms": 68.559, "count": 117, "max_ms": 90.646, "p95_ms": 88.928}, "visualize": {"avg_ms": 11.558, "count": 117, "max_ms": 14.959, "p95_ms": 12.635}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4018.121, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:42:21+09:00", "last_error_summary": "[h264 @ 0x556e59640680] error while decoding MB 41 19, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 103, "h264_decode_error_count": 106, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:42:21+09:00", "last_error_summary": "[h264 @ 0x556e59640680] error while decoding MB 41 19, bytestream -5", "last_exit_at": "2026-07-01T18:42:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.005, "last_started_at": "2026-07-01T18:40:31+09:00", "pid": 16329, "process": {"exists": false, "pid": 16329}, "restart_count": 118, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 119}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.452, "sample_count": 1148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57863, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:42:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:42:21+09:00", "last_frame_read_age_sec": 0.229, "last_frame_read_at": "2026-07-01T18:42:21+09:00", "mjpeg": {"age_sec": 0.225, "exists": true, "mtime": "2026-07-01T18:42:21.496143+09:00", "mtime_epoch": 1782898941.4961426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192081}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:42:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 886.445, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.954, "pushed": 3.954, "read": 3.92}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.048, "count": 118, "max_ms": 13.402, "p95_ms": 12.258}, "gstreamer_push": {"avg_ms": 0.941, "count": 118, "max_ms": 2.889, "p95_ms": 1.768}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.704, "count": 118, "max_ms": 69.723, "p95_ms": 62.845}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.175, "count": 118, "max_ms": 89.737, "p95_ms": 86.327}, "yolo11_640_fire_11class": {"avg_ms": 52.95, "count": 118, "max_ms": 71.246, "p95_ms": 68.958}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.703, "count": 118, "max_ms": 90.635, "p95_ms": 88.426}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.228, "count": 118, "max_ms": 322.091, "p95_ms": 273.789}, "read_interval": {"avg_ms": 255.036, "count": 117, "max_ms": 332.875, "p95_ms": 282.77}, "triton_wait": {"avg_ms": 68.498, "count": 118, "max_ms": 90.646, "p95_ms": 88.928}, "visualize": {"avg_ms": 11.552, "count": 118, "max_ms": 14.959, "p95_ms": 12.635}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.843, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4018.113, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 103, "h264_decode_error_count": 106, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:42:21+09:00", "last_error_summary": "[h264 @ 0x556e59640680] error while decoding MB 41 19, bytestream -5", "last_exit_at": "2026-07-01T18:42:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.008, "last_started_at": "2026-07-01T18:40:31+09:00", "pid": 16329, "process": {"exists": false, "pid": 16329}, "restart_count": 118, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 119}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.452, "sample_count": 1148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57863, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:42:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:42:21+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T18:42:21+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T18:42:21.496143+09:00", "mtime_epoch": 1782898941.4961426, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192081}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:42:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 886.445, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.954, "pushed": 3.954, "read": 3.92}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.048, "count": 118, "max_ms": 13.402, "p95_ms": 12.258}, "gstreamer_push": {"avg_ms": 0.941, "count": 118, "max_ms": 2.889, "p95_ms": 1.768}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.704, "count": 118, "max_ms": 69.723, "p95_ms": 62.845}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.175, "count": 118, "max_ms": 89.737, "p95_ms": 86.327}, "yolo11_640_fire_11class": {"avg_ms": 52.95, "count": 118, "max_ms": 71.246, "p95_ms": 68.958}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.703, "count": 118, "max_ms": 90.635, "p95_ms": 88.426}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.228, "count": 118, "max_ms": 322.091, "p95_ms": 273.789}, "read_interval": {"avg_ms": 255.036, "count": 117, "max_ms": 332.875, "p95_ms": 282.77}, "triton_wait": {"avg_ms": 68.498, "count": 118, "max_ms": 90.646, "p95_ms": 88.928}, "visualize": {"avg_ms": 11.552, "count": 118, "max_ms": 14.959, "p95_ms": 12.635}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.847, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 482}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4126.266, "limit_mb": 8796093022207.996}, "detail": {"pid": 16436, "restart_count": 119, "start_count": 120}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 103, "h264_decode_error_count": 106, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:42:21+09:00", "last_error_summary": "[h264 @ 0x556e59640680] error while decoding MB 41 19, bytestream -5", "last_exit_at": "2026-07-01T18:42:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:42:24+09:00", "pid": 16436, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.84, "name": "ffmpeg", "pid": 16436, "rss_mb": 141.84, "state": "S (sleeping)", "threads": 106}, "restart_count": 119, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 120}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.496, "sample_count": 1130, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 57863, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:42:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.875, "last_frame_pushed_at": "2026-07-01T18:42:21+09:00", "last_frame_read_age_sec": 5.102, "last_frame_read_at": "2026-07-01T18:42:21+09:00", "mjpeg": {"age_sec": 4.861, "exists": true, "mtime": "2026-07-01T18:42:21.732148+09:00", "mtime_epoch": 1782898941.7321477, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188449}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:42:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 891.438, "name": "celery", "pid": 237, "rss_mb": 886.445, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.278, "pushed": 3.278, "read": 3.245}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.174, "count": 97, "max_ms": 13.402, "p95_ms": 13.0}, "gstreamer_push": {"avg_ms": 0.957, "count": 98, "max_ms": 2.889, "p95_ms": 1.827}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.217, "count": 97, "max_ms": 69.723, "p95_ms": 62.845}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.824, "count": 97, "max_ms": 89.737, "p95_ms": 87.835}, "yolo11_640_fire_11class": {"avg_ms": 53.58, "count": 97, "max_ms": 71.246, "p95_ms": 69.165}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.539, "count": 97, "max_ms": 90.635, "p95_ms": 88.537}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.171, "count": 98, "max_ms": 322.091, "p95_ms": 274.213}, "read_interval": {"avg_ms": 258.104, "count": 97, "max_ms": 332.875, "p95_ms": 283.307}, "triton_wait": {"avg_ms": 69.264, "count": 97, "max_ms": 90.646, "p95_ms": 89.287}, "visualize": {"avg_ms": 11.659, "count": 97, "max_ms": 14.959, "p95_ms": 12.665}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 485}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4024.953, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559b1a875440] error while decoding MB 62 11, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 104, "h264_decode_error_count": 107, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:44:14+09:00", "last_error_summary": "[h264 @ 0x559b1a875440] error while decoding MB 62 11, bytestream -9", "last_exit_at": "2026-07-01T18:42:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.979, "last_started_at": "2026-07-01T18:42:24+09:00", "pid": 16436, "process": {"exists": false, "pid": 16436}, "restart_count": 119, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 120}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.452, "sample_count": 1143, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58293, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:44:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T18:44:14+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T18:44:14+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T18:44:14.530517+09:00", "mtime_epoch": 1782899054.530517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189146}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:44:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 894.906, "name": "celery", "pid": 237, "rss_mb": 892.059, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.104, "pushed": 4.104, "read": 4.104}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.485, "count": 122, "max_ms": 12.399, "p95_ms": 10.763}, "gstreamer_push": {"avg_ms": 0.951, "count": 123, "max_ms": 3.651, "p95_ms": 1.546}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.383, "count": 122, "max_ms": 70.66, "p95_ms": 58.88}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.183, "count": 122, "max_ms": 91.767, "p95_ms": 84.732}, "yolo11_640_fire_11class": {"avg_ms": 51.235, "count": 122, "max_ms": 71.182, "p95_ms": 66.584}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.391, "count": 122, "max_ms": 90.311, "p95_ms": 87.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.582, "count": 123, "max_ms": 292.258, "p95_ms": 266.856}, "read_interval": {"avg_ms": 245.116, "count": 123, "max_ms": 302.406, "p95_ms": 277.5}, "triton_wait": {"avg_ms": 65.963, "count": 122, "max_ms": 91.779, "p95_ms": 88.351}, "visualize": {"avg_ms": 11.064, "count": 122, "max_ms": 12.868, "p95_ms": 12.173}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 485}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4024.176, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:44:14+09:00", "last_error_summary": "[h264 @ 0x559b1a875440] error while decoding MB 62 11, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 104, "h264_decode_error_count": 107, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:44:14+09:00", "last_error_summary": "[h264 @ 0x559b1a875440] error while decoding MB 62 11, bytestream -9", "last_exit_at": "2026-07-01T18:44:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.183, "last_started_at": "2026-07-01T18:42:24+09:00", "pid": 16436, "process": {"exists": false, "pid": 16436}, "restart_count": 119, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 120}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.452, "sample_count": 1143, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:44:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T18:44:14+09:00", "last_frame_read_age_sec": 0.255, "last_frame_read_at": "2026-07-01T18:44:14+09:00", "mjpeg": {"age_sec": 0.245, "exists": true, "mtime": "2026-07-01T18:44:14.530517+09:00", "mtime_epoch": 1782899054.530517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189146}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:44:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 894.906, "name": "celery", "pid": 237, "rss_mb": 892.059, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.109, "pushed": 4.109, "read": 4.076}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.491, "count": 122, "max_ms": 12.399, "p95_ms": 10.763}, "gstreamer_push": {"avg_ms": 0.951, "count": 123, "max_ms": 3.651, "p95_ms": 1.546}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.366, "count": 122, "max_ms": 70.66, "p95_ms": 58.88}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.215, "count": 122, "max_ms": 91.767, "p95_ms": 84.732}, "yolo11_640_fire_11class": {"avg_ms": 51.258, "count": 122, "max_ms": 71.182, "p95_ms": 66.584}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.405, "count": 122, "max_ms": 90.311, "p95_ms": 87.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.756, "count": 123, "max_ms": 292.258, "p95_ms": 266.856}, "read_interval": {"avg_ms": 245.166, "count": 122, "max_ms": 302.406, "p95_ms": 277.5}, "triton_wait": {"avg_ms": 65.987, "count": 122, "max_ms": 91.779, "p95_ms": 88.351}, "visualize": {"avg_ms": 11.07, "count": 122, "max_ms": 12.868, "p95_ms": 12.173}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 485}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4024.168, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 104, "h264_decode_error_count": 107, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:44:14+09:00", "last_error_summary": "[h264 @ 0x559b1a875440] error while decoding MB 62 11, bytestream -9", "last_exit_at": "2026-07-01T18:44:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.186, "last_started_at": "2026-07-01T18:42:24+09:00", "pid": 16436, "process": {"exists": false, "pid": 16436}, "restart_count": 119, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 120}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.452, "sample_count": 1143, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:44:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T18:44:14+09:00", "last_frame_read_age_sec": 0.258, "last_frame_read_at": "2026-07-01T18:44:14+09:00", "mjpeg": {"age_sec": 0.248, "exists": true, "mtime": "2026-07-01T18:44:14.530517+09:00", "mtime_epoch": 1782899054.530517, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189146}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:44:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 894.906, "name": "celery", "pid": 237, "rss_mb": 892.059, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.109, "pushed": 4.109, "read": 4.075}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.491, "count": 122, "max_ms": 12.399, "p95_ms": 10.763}, "gstreamer_push": {"avg_ms": 0.951, "count": 123, "max_ms": 3.651, "p95_ms": 1.546}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.366, "count": 122, "max_ms": 70.66, "p95_ms": 58.88}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.215, "count": 122, "max_ms": 91.767, "p95_ms": 84.732}, "yolo11_640_fire_11class": {"avg_ms": 51.258, "count": 122, "max_ms": 71.182, "p95_ms": 66.584}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.405, "count": 122, "max_ms": 90.311, "p95_ms": 87.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.756, "count": 123, "max_ms": 292.258, "p95_ms": 266.856}, "read_interval": {"avg_ms": 245.166, "count": 122, "max_ms": 302.406, "p95_ms": 277.5}, "triton_wait": {"avg_ms": 65.987, "count": 122, "max_ms": 91.779, "p95_ms": 88.351}, "visualize": {"avg_ms": 11.07, "count": 122, "max_ms": 12.868, "p95_ms": 12.173}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 485}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4138.258, "limit_mb": 8796093022207.996}, "detail": {"pid": 16543, "restart_count": 120, "start_count": 121}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 104, "h264_decode_error_count": 107, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:44:14+09:00", "last_error_summary": "[h264 @ 0x559b1a875440] error while decoding MB 62 11, bytestream -9", "last_exit_at": "2026-07-01T18:44:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:44:17+09:00", "pid": 16543, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.891, "name": "ffmpeg", "pid": 16543, "rss_mb": 144.891, "state": "S (sleeping)", "threads": 106}, "restart_count": 120, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 121}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.219, "sample_count": 1124, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:44:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.924, "last_frame_pushed_at": "2026-07-01T18:44:14+09:00", "last_frame_read_age_sec": 5.175, "last_frame_read_at": "2026-07-01T18:44:14+09:00", "mjpeg": {"age_sec": 4.905, "exists": true, "mtime": "2026-07-01T18:44:14.790523+09:00", "mtime_epoch": 1782899054.7905226, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189805}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:44:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 894.906, "name": "celery", "pid": 237, "rss_mb": 892.059, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.431, "pushed": 3.431, "read": 3.397}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.457, "count": 102, "max_ms": 12.399, "p95_ms": 10.815}, "gstreamer_push": {"avg_ms": 0.939, "count": 102, "max_ms": 3.651, "p95_ms": 0.974}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.105, "count": 102, "max_ms": 70.66, "p95_ms": 59.356}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.062, "count": 102, "max_ms": 91.767, "p95_ms": 84.732}, "yolo11_640_fire_11class": {"avg_ms": 51.014, "count": 102, "max_ms": 71.182, "p95_ms": 66.624}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.236, "count": 102, "max_ms": 90.311, "p95_ms": 87.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.551, "count": 102, "max_ms": 292.258, "p95_ms": 271.753}, "read_interval": {"avg_ms": 245.953, "count": 101, "max_ms": 302.406, "p95_ms": 282.19}, "triton_wait": {"avg_ms": 65.829, "count": 102, "max_ms": 91.779, "p95_ms": 88.351}, "visualize": {"avg_ms": 11.111, "count": 102, "max_ms": 12.868, "p95_ms": 12.253}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.728, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4026.309, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557b88623340] error while decoding MB 72 36, bytestream -28", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 105, "h264_decode_error_count": 108, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:46:08+09:00", "last_error_summary": "[h264 @ 0x557b88623340] error while decoding MB 72 36, bytestream -28", "last_exit_at": "2026-07-01T18:44:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.901, "last_started_at": "2026-07-01T18:44:17+09:00", "pid": 16543, "process": {"exists": false, "pid": 16543}, "restart_count": 120, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 121}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.369, "sample_count": 1145, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58722, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:46:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T18:46:08+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T18:46:08+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T18:46:08.548869+09:00", "mtime_epoch": 1782899168.548869, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193445}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:46:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.07, "name": "celery", "pid": 237, "rss_mb": 894.57, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.892, "pushed": 3.892, "read": 3.892}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.065, "count": 116, "max_ms": 12.774, "p95_ms": 12.599}, "gstreamer_push": {"avg_ms": 0.901, "count": 116, "max_ms": 3.057, "p95_ms": 0.985}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.652, "count": 116, "max_ms": 73.162, "p95_ms": 63.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.858, "count": 116, "max_ms": 95.259, "p95_ms": 91.79}, "yolo11_640_fire_11class": {"avg_ms": 54.367, "count": 116, "max_ms": 72.408, "p95_ms": 68.97}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.654, "count": 116, "max_ms": 94.001, "p95_ms": 90.538}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.347, "count": 116, "max_ms": 349.846, "p95_ms": 279.55}, "read_interval": {"avg_ms": 258.659, "count": 116, "max_ms": 361.027, "p95_ms": 289.523}, "triton_wait": {"avg_ms": 71.301, "count": 116, "max_ms": 95.271, "p95_ms": 91.893}, "visualize": {"avg_ms": 11.679, "count": 116, "max_ms": 15.092, "p95_ms": 14.682}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.808, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4027.906, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:46:08+09:00", "last_error_summary": "[h264 @ 0x557b88623340] error while decoding MB 72 36, bytestream -28", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 105, "h264_decode_error_count": 108, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:46:08+09:00", "last_error_summary": "[h264 @ 0x557b88623340] error while decoding MB 72 36, bytestream -28", "last_exit_at": "2026-07-01T18:46:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.103, "last_started_at": "2026-07-01T18:44:17+09:00", "pid": 16543, "process": {"exists": false, "pid": 16543}, "restart_count": 120, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 121}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.369, "sample_count": 1145, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58723, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:46:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:46:08+09:00", "last_frame_read_age_sec": 0.252, "last_frame_read_at": "2026-07-01T18:46:08+09:00", "mjpeg": {"age_sec": 0.248, "exists": true, "mtime": "2026-07-01T18:46:08.548869+09:00", "mtime_epoch": 1782899168.548869, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193445}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:46:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.07, "name": "celery", "pid": 237, "rss_mb": 894.57, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.867, "pushed": 3.867, "read": 3.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.083, "count": 116, "max_ms": 12.774, "p95_ms": 12.599}, "gstreamer_push": {"avg_ms": 0.901, "count": 116, "max_ms": 3.057, "p95_ms": 0.985}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.696, "count": 116, "max_ms": 73.162, "p95_ms": 63.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.869, "count": 116, "max_ms": 95.259, "p95_ms": 91.79}, "yolo11_640_fire_11class": {"avg_ms": 54.36, "count": 116, "max_ms": 72.408, "p95_ms": 68.97}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.679, "count": 116, "max_ms": 94.001, "p95_ms": 90.538}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.348, "count": 116, "max_ms": 349.846, "p95_ms": 279.55}, "read_interval": {"avg_ms": 258.659, "count": 116, "max_ms": 361.027, "p95_ms": 289.523}, "triton_wait": {"avg_ms": 71.312, "count": 116, "max_ms": 95.271, "p95_ms": 91.893}, "visualize": {"avg_ms": 11.664, "count": 116, "max_ms": 15.092, "p95_ms": 14.682}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4027.898, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 105, "h264_decode_error_count": 108, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:46:08+09:00", "last_error_summary": "[h264 @ 0x557b88623340] error while decoding MB 72 36, bytestream -28", "last_exit_at": "2026-07-01T18:46:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.106, "last_started_at": "2026-07-01T18:44:17+09:00", "pid": 16543, "process": {"exists": false, "pid": 16543}, "restart_count": 120, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 121}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.369, "sample_count": 1145, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58723, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:46:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:46:08+09:00", "last_frame_read_age_sec": 0.256, "last_frame_read_at": "2026-07-01T18:46:08+09:00", "mjpeg": {"age_sec": 0.251, "exists": true, "mtime": "2026-07-01T18:46:08.548869+09:00", "mtime_epoch": 1782899168.548869, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193445}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:46:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.07, "name": "celery", "pid": 237, "rss_mb": 894.57, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.898, "pushed": 3.898, "read": 3.865}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.083, "count": 116, "max_ms": 12.774, "p95_ms": 12.599}, "gstreamer_push": {"avg_ms": 0.901, "count": 116, "max_ms": 3.057, "p95_ms": 0.985}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.696, "count": 116, "max_ms": 73.162, "p95_ms": 63.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.869, "count": 116, "max_ms": 95.259, "p95_ms": 91.79}, "yolo11_640_fire_11class": {"avg_ms": 54.36, "count": 116, "max_ms": 72.408, "p95_ms": 68.97}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.679, "count": 116, "max_ms": 94.001, "p95_ms": 90.538}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.348, "count": 116, "max_ms": 349.846, "p95_ms": 279.55}, "read_interval": {"avg_ms": 258.671, "count": 115, "max_ms": 361.027, "p95_ms": 289.523}, "triton_wait": {"avg_ms": 71.312, "count": 116, "max_ms": 95.271, "p95_ms": 91.893}, "visualize": {"avg_ms": 11.664, "count": 116, "max_ms": 15.092, "p95_ms": 14.682}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.757, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4135.609, "limit_mb": 8796093022207.996}, "detail": {"pid": 16650, "restart_count": 121, "start_count": 122}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 105, "h264_decode_error_count": 108, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:46:08+09:00", "last_error_summary": "[h264 @ 0x557b88623340] error while decoding MB 72 36, bytestream -28", "last_exit_at": "2026-07-01T18:46:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:46:10+09:00", "pid": 16650, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.773, "name": "ffmpeg", "pid": 16650, "rss_mb": 138.773, "state": "S (sleeping)", "threads": 106}, "restart_count": 121, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 122}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.369, "sample_count": 1128, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 58723, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:46:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.164, "last_frame_pushed_at": "2026-07-01T18:46:08+09:00", "last_frame_read_age_sec": 4.414, "last_frame_read_at": "2026-07-01T18:46:08+09:00", "mjpeg": {"age_sec": 4.146, "exists": true, "mtime": "2026-07-01T18:46:08.812874+09:00", "mtime_epoch": 1782899168.8128743, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193502}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:46:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.07, "name": "celery", "pid": 237, "rss_mb": 894.57, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.384, "pushed": 3.384, "read": 3.35}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.928, "count": 101, "max_ms": 12.495, "p95_ms": 11.134}, "gstreamer_push": {"avg_ms": 0.906, "count": 101, "max_ms": 3.057, "p95_ms": 0.981}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.767, "count": 101, "max_ms": 73.162, "p95_ms": 63.523}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.57, "count": 101, "max_ms": 95.259, "p95_ms": 91.241}, "yolo11_640_fire_11class": {"avg_ms": 53.882, "count": 101, "max_ms": 72.408, "p95_ms": 68.702}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.256, "count": 101, "max_ms": 94.001, "p95_ms": 90.538}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.377, "count": 101, "max_ms": 349.846, "p95_ms": 277.92}, "read_interval": {"avg_ms": 256.637, "count": 100, "max_ms": 361.027, "p95_ms": 287.542}, "triton_wait": {"avg_ms": 70.931, "count": 101, "max_ms": 95.271, "p95_ms": 91.893}, "visualize": {"avg_ms": 11.434, "count": 101, "max_ms": 13.284, "p95_ms": 12.673}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.847, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 493}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4024.57, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5626b3ac9540] error while decoding MB 74 6, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 106, "h264_decode_error_count": 109, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:48:05+09:00", "last_error_summary": "[h264 @ 0x5626b3ac9540] error while decoding MB 74 6, bytestream -14", "last_exit_at": "2026-07-01T18:46:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.628, "last_started_at": "2026-07-01T18:46:10+09:00", "pid": 16650, "process": {"exists": false, "pid": 16650}, "restart_count": 121, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 122}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.608, "sample_count": 1150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:48:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.055, "last_frame_pushed_at": "2026-07-01T18:48:05+09:00", "last_frame_read_age_sec": 0.044, "last_frame_read_at": "2026-07-01T18:48:05+09:00", "mjpeg": {"age_sec": 0.034, "exists": true, "mtime": "2026-07-01T18:48:05.551245+09:00", "mtime_epoch": 1782899285.5512455, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192407}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:48:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.277, "name": "celery", "pid": 237, "rss_mb": 892.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.956, "pushed": 3.956, "read": 3.956}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.888, "count": 118, "max_ms": 16.4, "p95_ms": 11.028}, "gstreamer_push": {"avg_ms": 0.882, "count": 118, "max_ms": 2.195, "p95_ms": 0.931}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.632, "count": 118, "max_ms": 71.828, "p95_ms": 62.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.618, "count": 118, "max_ms": 91.837, "p95_ms": 88.65}, "yolo11_640_fire_11class": {"avg_ms": 53.909, "count": 118, "max_ms": 71.873, "p95_ms": 69.142}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.788, "count": 118, "max_ms": 91.611, "p95_ms": 88.698}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.765, "count": 118, "max_ms": 331.54, "p95_ms": 272.779}, "read_interval": {"avg_ms": 254.35, "count": 118, "max_ms": 340.617, "p95_ms": 281.858}, "triton_wait": {"avg_ms": 70.245, "count": 118, "max_ms": 91.85, "p95_ms": 89.453}, "visualize": {"avg_ms": 11.477, "count": 118, "max_ms": 14.892, "p95_ms": 12.818}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.831, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 493}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4023.508, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:48:05+09:00", "last_error_summary": "[h264 @ 0x5626b3ac9540] error while decoding MB 74 6, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 106, "h264_decode_error_count": 109, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:48:05+09:00", "last_error_summary": "[h264 @ 0x5626b3ac9540] error while decoding MB 74 6, bytestream -14", "last_exit_at": "2026-07-01T18:48:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.816, "last_started_at": "2026-07-01T18:46:10+09:00", "pid": 16650, "process": {"exists": false, "pid": 16650}, "restart_count": 121, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 122}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.608, "sample_count": 1150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59167, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:48:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T18:48:05+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-01T18:48:05+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T18:48:05.551245+09:00", "mtime_epoch": 1782899285.5512455, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192407}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:48:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.277, "name": "celery", "pid": 237, "rss_mb": 892.07, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.962, "pushed": 3.962, "read": 3.929}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.882, "count": 118, "max_ms": 16.4, "p95_ms": 11.028}, "gstreamer_push": {"avg_ms": 0.882, "count": 118, "max_ms": 2.195, "p95_ms": 0.931}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.581, "count": 118, "max_ms": 71.828, "p95_ms": 62.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.602, "count": 118, "max_ms": 91.837, "p95_ms": 88.65}, "yolo11_640_fire_11class": {"avg_ms": 53.904, "count": 118, "max_ms": 71.873, "p95_ms": 69.142}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.784, "count": 118, "max_ms": 91.611, "p95_ms": 88.698}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.767, "count": 118, "max_ms": 331.54, "p95_ms": 272.779}, "read_interval": {"avg_ms": 254.495, "count": 117, "max_ms": 340.617, "p95_ms": 281.858}, "triton_wait": {"avg_ms": 70.229, "count": 118, "max_ms": 91.85, "p95_ms": 89.453}, "visualize": {"avg_ms": 11.479, "count": 118, "max_ms": 14.892, "p95_ms": 12.818}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.782, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 493}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4023.5, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 106, "h264_decode_error_count": 109, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:48:05+09:00", "last_error_summary": "[h264 @ 0x5626b3ac9540] error while decoding MB 74 6, bytestream -14", "last_exit_at": "2026-07-01T18:48:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.819, "last_started_at": "2026-07-01T18:46:10+09:00", "pid": 16650, "process": {"exists": false, "pid": 16650}, "restart_count": 121, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 122}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.608, "sample_count": 1150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59167, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:48:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T18:48:05+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-01T18:48:05+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-01T18:48:05.551245+09:00", "mtime_epoch": 1782899285.5512455, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192407}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:48:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.277, "name": "celery", "pid": 237, "rss_mb": 892.07, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.962, "pushed": 3.962, "read": 3.928}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.882, "count": 118, "max_ms": 16.4, "p95_ms": 11.028}, "gstreamer_push": {"avg_ms": 0.882, "count": 118, "max_ms": 2.195, "p95_ms": 0.931}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.581, "count": 118, "max_ms": 71.828, "p95_ms": 62.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.602, "count": 118, "max_ms": 91.837, "p95_ms": 88.65}, "yolo11_640_fire_11class": {"avg_ms": 53.904, "count": 118, "max_ms": 71.873, "p95_ms": 69.142}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.784, "count": 118, "max_ms": 91.611, "p95_ms": 88.698}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.767, "count": 118, "max_ms": 331.54, "p95_ms": 272.779}, "read_interval": {"avg_ms": 254.495, "count": 117, "max_ms": 340.617, "p95_ms": 281.858}, "triton_wait": {"avg_ms": 70.229, "count": 118, "max_ms": 91.85, "p95_ms": 89.453}, "visualize": {"avg_ms": 11.479, "count": 118, "max_ms": 14.892, "p95_ms": 12.818}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.785, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 493}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4131.684, "limit_mb": 8796093022207.996}, "detail": {"pid": 16757, "restart_count": 122, "start_count": 123}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 106, "h264_decode_error_count": 109, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:48:05+09:00", "last_error_summary": "[h264 @ 0x5626b3ac9540] error while decoding MB 74 6, bytestream -14", "last_exit_at": "2026-07-01T18:48:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:48:07+09:00", "pid": 16757, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.832, "name": "ffmpeg", "pid": 16757, "rss_mb": 136.832, "state": "S (sleeping)", "threads": 106}, "restart_count": 122, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 123}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.573, "sample_count": 1134, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59167, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:48:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.195, "last_frame_pushed_at": "2026-07-01T18:48:05+09:00", "last_frame_read_age_sec": 4.424, "last_frame_read_at": "2026-07-01T18:48:05+09:00", "mjpeg": {"age_sec": 4.178, "exists": true, "mtime": "2026-07-01T18:48:05.787250+09:00", "mtime_epoch": 1782899285.7872503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192453}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:48:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.277, "name": "celery", "pid": 237, "rss_mb": 892.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.381, "pushed": 3.381, "read": 3.348}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.004, "count": 100, "max_ms": 16.4, "p95_ms": 11.022}, "gstreamer_push": {"avg_ms": 0.882, "count": 101, "max_ms": 2.195, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.796, "count": 100, "max_ms": 71.828, "p95_ms": 62.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.057, "count": 100, "max_ms": 90.026, "p95_ms": 88.328}, "yolo11_640_fire_11class": {"avg_ms": 54.34, "count": 100, "max_ms": 71.873, "p95_ms": 69.142}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.465, "count": 100, "max_ms": 91.611, "p95_ms": 88.698}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.052, "count": 101, "max_ms": 331.54, "p95_ms": 272.779}, "read_interval": {"avg_ms": 256.841, "count": 100, "max_ms": 340.617, "p95_ms": 281.858}, "triton_wait": {"avg_ms": 70.795, "count": 100, "max_ms": 91.625, "p95_ms": 89.097}, "visualize": {"avg_ms": 11.565, "count": 100, "max_ms": 14.892, "p95_ms": 12.818}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 496}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4021.961, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:48:05+09:00", "last_error_summary": "[h264 @ 0x5626b3ac9540] error while decoding MB 74 6, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 106, "h264_decode_error_count": 109, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:48:05+09:00", "last_error_summary": "[h264 @ 0x5626b3ac9540] error while decoding MB 74 6, bytestream -14", "last_exit_at": "2026-07-01T18:50:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.424, "last_started_at": "2026-07-01T18:48:07+09:00", "pid": 16757, "process": {"exists": false, "pid": 16757}, "restart_count": 122, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 123}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.92, "sample_count": 1135, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59617, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:50:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:50:07+09:00", "last_frame_read_age_sec": 0.249, "last_frame_read_at": "2026-07-01T18:50:07+09:00", "mjpeg": {"age_sec": 0.243, "exists": true, "mtime": "2026-07-01T18:50:07.145684+09:00", "mtime_epoch": 1782899407.1456842, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:50:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.277, "name": "celery", "pid": 237, "rss_mb": 889.246, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.767, "pushed": 3.767, "read": 3.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.995, "count": 113, "max_ms": 11.661, "p95_ms": 10.988}, "gstreamer_push": {"avg_ms": 0.895, "count": 113, "max_ms": 1.903, "p95_ms": 0.925}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.07, "count": 113, "max_ms": 70.686, "p95_ms": 68.943}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.585, "count": 113, "max_ms": 92.438, "p95_ms": 89.608}, "yolo11_640_fire_11class": {"avg_ms": 55.761, "count": 113, "max_ms": 71.929, "p95_ms": 69.27}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.492, "count": 113, "max_ms": 94.173, "p95_ms": 89.971}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.874, "count": 113, "max_ms": 347.873, "p95_ms": 278.412}, "read_interval": {"avg_ms": 265.475, "count": 113, "max_ms": 358.22, "p95_ms": 288.754}, "triton_wait": {"avg_ms": 73.57, "count": 113, "max_ms": 94.186, "p95_ms": 90.395}, "visualize": {"avg_ms": 11.797, "count": 113, "max_ms": 14.627, "p95_ms": 12.635}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 496}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4021.953, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 106, "h264_decode_error_count": 109, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:48:05+09:00", "last_error_summary": "[h264 @ 0x5626b3ac9540] error while decoding MB 74 6, bytestream -14", "last_exit_at": "2026-07-01T18:50:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 119.428, "last_started_at": "2026-07-01T18:48:07+09:00", "pid": 16757, "process": {"exists": false, "pid": 16757}, "restart_count": 122, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 123}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.92, "sample_count": 1135, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59617, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:50:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:50:07+09:00", "last_frame_read_age_sec": 0.252, "last_frame_read_at": "2026-07-01T18:50:07+09:00", "mjpeg": {"age_sec": 0.246, "exists": true, "mtime": "2026-07-01T18:50:07.145684+09:00", "mtime_epoch": 1782899407.1456842, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:50:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.277, "name": "celery", "pid": 237, "rss_mb": 889.246, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.796, "pushed": 3.796, "read": 3.762}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.995, "count": 113, "max_ms": 11.661, "p95_ms": 10.988}, "gstreamer_push": {"avg_ms": 0.895, "count": 113, "max_ms": 1.903, "p95_ms": 0.925}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.07, "count": 113, "max_ms": 70.686, "p95_ms": 68.943}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.585, "count": 113, "max_ms": 92.438, "p95_ms": 89.608}, "yolo11_640_fire_11class": {"avg_ms": 55.761, "count": 113, "max_ms": 71.929, "p95_ms": 69.27}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.492, "count": 113, "max_ms": 94.173, "p95_ms": 89.971}}, "truncated_count": 0}, "processing_total": {"avg_ms": 255.874, "count": 113, "max_ms": 347.873, "p95_ms": 278.412}, "read_interval": {"avg_ms": 265.628, "count": 112, "max_ms": 358.22, "p95_ms": 288.754}, "triton_wait": {"avg_ms": 73.57, "count": 113, "max_ms": 94.186, "p95_ms": 90.395}, "visualize": {"avg_ms": 11.797, "count": 113, "max_ms": 14.627, "p95_ms": 12.635}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.77, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 496}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4129.707, "limit_mb": 8796093022207.996}, "detail": {"pid": 16864, "restart_count": 123, "start_count": 124}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 106, "h264_decode_error_count": 109, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:48:05+09:00", "last_error_summary": "[h264 @ 0x5626b3ac9540] error while decoding MB 74 6, bytestream -14", "last_exit_at": "2026-07-01T18:50:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:50:09+09:00", "pid": 16864, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.664, "name": "ffmpeg", "pid": 16864, "rss_mb": 136.664, "state": "S (sleeping)", "threads": 106}, "restart_count": 123, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 124}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.412, "sample_count": 1118, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59617, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:50:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.297, "last_frame_pushed_at": "2026-07-01T18:50:07+09:00", "last_frame_read_age_sec": 4.544, "last_frame_read_at": "2026-07-01T18:50:07+09:00", "mjpeg": {"age_sec": 4.281, "exists": true, "mtime": "2026-07-01T18:50:07.401689+09:00", "mtime_epoch": 1782899407.4016893, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193820}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:50:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.277, "name": "celery", "pid": 237, "rss_mb": 889.246, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.205, "pushed": 3.205, "read": 3.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.058, "count": 95, "max_ms": 11.661, "p95_ms": 10.988}, "gstreamer_push": {"avg_ms": 0.901, "count": 96, "max_ms": 1.903, "p95_ms": 0.945}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.793, "count": 95, "max_ms": 70.686, "p95_ms": 68.981}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.725, "count": 95, "max_ms": 92.438, "p95_ms": 90.22}, "yolo11_640_fire_11class": {"avg_ms": 56.514, "count": 95, "max_ms": 71.929, "p95_ms": 69.412}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.622, "count": 95, "max_ms": 94.173, "p95_ms": 90.25}}, "truncated_count": 0}, "processing_total": {"avg_ms": 260.37, "count": 96, "max_ms": 347.873, "p95_ms": 278.731}, "read_interval": {"avg_ms": 270.203, "count": 95, "max_ms": 358.22, "p95_ms": 289.249}, "triton_wait": {"avg_ms": 74.785, "count": 95, "max_ms": 94.186, "p95_ms": 90.441}, "visualize": {"avg_ms": 11.93, "count": 95, "max_ms": 14.627, "p95_ms": 12.635}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 500}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4025.832, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561c3e9112c0] error while decoding MB 37 43, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 107, "h264_decode_error_count": 110, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:51:59+09:00", "last_error_summary": "[h264 @ 0x561c3e9112c0] error while decoding MB 37 43, bytestream -7", "last_exit_at": "2026-07-01T18:50:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.233, "last_started_at": "2026-07-01T18:50:09+09:00", "pid": 16864, "process": {"exists": false, "pid": 16864}, "restart_count": 123, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 124}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.941, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60049, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:51:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.05, "last_frame_pushed_at": "2026-07-01T18:51:59+09:00", "last_frame_read_age_sec": 0.042, "last_frame_read_at": "2026-07-01T18:51:59+09:00", "mjpeg": {"age_sec": 0.027, "exists": true, "mtime": "2026-07-01T18:51:59.887923+09:00", "mtime_epoch": 1782899519.887923, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:51:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.277, "name": "celery", "pid": 237, "rss_mb": 889.738, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.094, "pushed": 4.094, "read": 4.094}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.449, "count": 121, "max_ms": 13.126, "p95_ms": 10.757}, "gstreamer_push": {"avg_ms": 0.939, "count": 122, "max_ms": 3.044, "p95_ms": 1.727}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.477, "count": 121, "max_ms": 72.433, "p95_ms": 60.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.574, "count": 121, "max_ms": 93.521, "p95_ms": 83.832}, "yolo11_640_fire_11class": {"avg_ms": 55.276, "count": 121, "max_ms": 75.014, "p95_ms": 71.018}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.023, "count": 121, "max_ms": 92.5, "p95_ms": 85.036}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.353, "count": 122, "max_ms": 299.241, "p95_ms": 262.285}, "read_interval": {"avg_ms": 246.285, "count": 122, "max_ms": 308.039, "p95_ms": 273.038}, "triton_wait": {"avg_ms": 67.262, "count": 121, "max_ms": 93.533, "p95_ms": 86.74}, "visualize": {"avg_ms": 11.298, "count": 121, "max_ms": 16.651, "p95_ms": 13.025}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.799, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 500}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4021.867, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:51:59+09:00", "last_error_summary": "[h264 @ 0x561c3e9112c0] error while decoding MB 37 43, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 107, "h264_decode_error_count": 110, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:51:59+09:00", "last_error_summary": "[h264 @ 0x561c3e9112c0] error while decoding MB 37 43, bytestream -7", "last_exit_at": "2026-07-01T18:52:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.482, "last_started_at": "2026-07-01T18:50:09+09:00", "pid": 16864, "process": {"exists": false, "pid": 16864}, "restart_count": 123, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 124}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.401, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:51:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:52:00+09:00", "last_frame_read_age_sec": 0.29, "last_frame_read_at": "2026-07-01T18:51:59+09:00", "mjpeg": {"age_sec": 0.276, "exists": true, "mtime": "2026-07-01T18:51:59.887923+09:00", "mtime_epoch": 1782899519.887923, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:52:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.277, "name": "celery", "pid": 237, "rss_mb": 886.73, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.093, "pushed": 4.093, "read": 4.06}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.445, "count": 122, "max_ms": 13.126, "p95_ms": 10.757}, "gstreamer_push": {"avg_ms": 0.921, "count": 122, "max_ms": 1.892, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.407, "count": 122, "max_ms": 72.433, "p95_ms": 60.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.516, "count": 122, "max_ms": 93.521, "p95_ms": 83.832}, "yolo11_640_fire_11class": {"avg_ms": 55.223, "count": 122, "max_ms": 75.014, "p95_ms": 71.018}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.987, "count": 122, "max_ms": 92.5, "p95_ms": 85.036}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.326, "count": 122, "max_ms": 299.241, "p95_ms": 262.285}, "read_interval": {"avg_ms": 245.834, "count": 121, "max_ms": 308.039, "p95_ms": 264.796}, "triton_wait": {"avg_ms": 67.217, "count": 122, "max_ms": 93.533, "p95_ms": 86.74}, "visualize": {"avg_ms": 11.295, "count": 122, "max_ms": 16.651, "p95_ms": 13.025}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.806, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 500}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4021.859, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 107, "h264_decode_error_count": 110, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:51:59+09:00", "last_error_summary": "[h264 @ 0x561c3e9112c0] error while decoding MB 37 43, bytestream -7", "last_exit_at": "2026-07-01T18:52:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.485, "last_started_at": "2026-07-01T18:50:09+09:00", "pid": 16864, "process": {"exists": false, "pid": 16864}, "restart_count": 123, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 124}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.401, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:51:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:52:00+09:00", "last_frame_read_age_sec": 0.294, "last_frame_read_at": "2026-07-01T18:51:59+09:00", "mjpeg": {"age_sec": 0.279, "exists": true, "mtime": "2026-07-01T18:51:59.887923+09:00", "mtime_epoch": 1782899519.887923, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:52:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.277, "name": "celery", "pid": 237, "rss_mb": 886.73, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.093, "pushed": 4.093, "read": 4.059}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.445, "count": 122, "max_ms": 13.126, "p95_ms": 10.757}, "gstreamer_push": {"avg_ms": 0.921, "count": 122, "max_ms": 1.892, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.407, "count": 122, "max_ms": 72.433, "p95_ms": 60.296}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.516, "count": 122, "max_ms": 93.521, "p95_ms": 83.832}, "yolo11_640_fire_11class": {"avg_ms": 55.223, "count": 122, "max_ms": 75.014, "p95_ms": 71.018}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.987, "count": 122, "max_ms": 92.5, "p95_ms": 85.036}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.326, "count": 122, "max_ms": 299.241, "p95_ms": 262.285}, "read_interval": {"avg_ms": 245.834, "count": 121, "max_ms": 308.039, "p95_ms": 264.796}, "triton_wait": {"avg_ms": 67.217, "count": 122, "max_ms": 93.533, "p95_ms": 86.74}, "visualize": {"avg_ms": 11.295, "count": 122, "max_ms": 16.651, "p95_ms": 13.025}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.809, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 500}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4127.176, "limit_mb": 8796093022207.996}, "detail": {"pid": 16971, "restart_count": 124, "start_count": 125}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 107, "h264_decode_error_count": 110, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:51:59+09:00", "last_error_summary": "[h264 @ 0x561c3e9112c0] error while decoding MB 37 43, bytestream -7", "last_exit_at": "2026-07-01T18:52:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:52:02+09:00", "pid": 16971, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.391, "name": "ffmpeg", "pid": 16971, "rss_mb": 136.391, "state": "S (sleeping)", "threads": 106}, "restart_count": 124, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 125}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.558, "sample_count": 1123, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:51:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.726, "last_frame_pushed_at": "2026-07-01T18:52:00+09:00", "last_frame_read_age_sec": 5.014, "last_frame_read_at": "2026-07-01T18:51:59+09:00", "mjpeg": {"age_sec": 4.707, "exists": true, "mtime": "2026-07-01T18:52:00.179929+09:00", "mtime_epoch": 1782899520.1799288, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191766}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:52:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.277, "name": "celery", "pid": 237, "rss_mb": 886.73, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.443, "pushed": 3.443, "read": 3.41}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.407, "count": 102, "max_ms": 13.126, "p95_ms": 10.733}, "gstreamer_push": {"avg_ms": 0.927, "count": 103, "max_ms": 1.892, "p95_ms": 1.727}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.896, "count": 102, "max_ms": 72.433, "p95_ms": 60.92}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.791, "count": 102, "max_ms": 93.521, "p95_ms": 85.021}, "yolo11_640_fire_11class": {"avg_ms": 55.903, "count": 102, "max_ms": 75.014, "p95_ms": 71.774}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.252, "count": 102, "max_ms": 92.5, "p95_ms": 85.306}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.942, "count": 103, "max_ms": 299.241, "p95_ms": 266.467}, "read_interval": {"avg_ms": 246.252, "count": 102, "max_ms": 308.039, "p95_ms": 273.038}, "triton_wait": {"avg_ms": 67.567, "count": 102, "max_ms": 93.533, "p95_ms": 87.352}, "visualize": {"avg_ms": 11.299, "count": 102, "max_ms": 16.651, "p95_ms": 12.72}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 504}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4023.266, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:51:59+09:00", "last_error_summary": "[h264 @ 0x561c3e9112c0] error while decoding MB 37 43, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 107, "h264_decode_error_count": 110, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:51:59+09:00", "last_error_summary": "[h264 @ 0x561c3e9112c0] error while decoding MB 37 43, bytestream -7", "last_exit_at": "2026-07-01T18:53:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.319, "last_started_at": "2026-07-01T18:52:02+09:00", "pid": 16971, "process": {"exists": false, "pid": 16971}, "restart_count": 124, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 125}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.072, "sample_count": 1149, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60485, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:53:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T18:53:53+09:00", "last_frame_read_age_sec": 0.223, "last_frame_read_at": "2026-07-01T18:53:52+09:00", "mjpeg": {"age_sec": 0.211, "exists": true, "mtime": "2026-07-01T18:53:52.994152+09:00", "mtime_epoch": 1782899632.9941516, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:53:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 889.98, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.008, "pushed": 4.008, "read": 3.975}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.522, "count": 119, "max_ms": 13.424, "p95_ms": 10.939}, "gstreamer_push": {"avg_ms": 0.986, "count": 120, "max_ms": 3.122, "p95_ms": 1.811}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.839, "count": 119, "max_ms": 69.36, "p95_ms": 67.267}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.737, "count": 119, "max_ms": 93.293, "p95_ms": 89.695}, "yolo11_640_fire_11class": {"avg_ms": 55.291, "count": 119, "max_ms": 75.347, "p95_ms": 72.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.314, "count": 119, "max_ms": 92.089, "p95_ms": 90.033}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.652, "count": 120, "max_ms": 318.725, "p95_ms": 273.234}, "read_interval": {"avg_ms": 251.631, "count": 119, "max_ms": 330.194, "p95_ms": 282.804}, "triton_wait": {"avg_ms": 69.443, "count": 119, "max_ms": 93.306, "p95_ms": 90.478}, "visualize": {"avg_ms": 11.281, "count": 119, "max_ms": 16.016, "p95_ms": 13.44}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 504}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4023.094, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 107, "h264_decode_error_count": 110, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:51:59+09:00", "last_error_summary": "[h264 @ 0x561c3e9112c0] error while decoding MB 37 43, bytestream -7", "last_exit_at": "2026-07-01T18:53:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.322, "last_started_at": "2026-07-01T18:52:02+09:00", "pid": 16971, "process": {"exists": false, "pid": 16971}, "restart_count": 124, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 125}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.072, "sample_count": 1149, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60485, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:53:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T18:53:53+09:00", "last_frame_read_age_sec": 0.226, "last_frame_read_at": "2026-07-01T18:53:52+09:00", "mjpeg": {"age_sec": 0.214, "exists": true, "mtime": "2026-07-01T18:53:52.994152+09:00", "mtime_epoch": 1782899632.9941516, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:53:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 889.98, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.008, "pushed": 4.008, "read": 3.975}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.522, "count": 119, "max_ms": 13.424, "p95_ms": 10.939}, "gstreamer_push": {"avg_ms": 0.986, "count": 120, "max_ms": 3.122, "p95_ms": 1.811}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.839, "count": 119, "max_ms": 69.36, "p95_ms": 67.267}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.737, "count": 119, "max_ms": 93.293, "p95_ms": 89.695}, "yolo11_640_fire_11class": {"avg_ms": 55.291, "count": 119, "max_ms": 75.347, "p95_ms": 72.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.314, "count": 119, "max_ms": 92.089, "p95_ms": 90.033}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.652, "count": 120, "max_ms": 318.725, "p95_ms": 273.234}, "read_interval": {"avg_ms": 251.631, "count": 119, "max_ms": 330.194, "p95_ms": 282.804}, "triton_wait": {"avg_ms": 69.443, "count": 119, "max_ms": 93.306, "p95_ms": 90.478}, "visualize": {"avg_ms": 11.281, "count": 119, "max_ms": 16.016, "p95_ms": 13.44}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 504}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4144.25, "limit_mb": 8796093022207.996}, "detail": {"pid": 17078, "restart_count": 125, "start_count": 126}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 107, "h264_decode_error_count": 110, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:51:59+09:00", "last_error_summary": "[h264 @ 0x561c3e9112c0] error while decoding MB 37 43, bytestream -7", "last_exit_at": "2026-07-01T18:53:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:53:55+09:00", "pid": 17078, "process": {"exists": true, "fd_count": 5, "hwm_mb": 150.312, "name": "ffmpeg", "pid": 17078, "rss_mb": 150.312, "state": "S (sleeping)", "threads": 106}, "restart_count": 125, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 126}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.441, "sample_count": 1132, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60485, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:53:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.571, "last_frame_pushed_at": "2026-07-01T18:53:53+09:00", "last_frame_read_age_sec": 4.79, "last_frame_read_at": "2026-07-01T18:53:52+09:00", "mjpeg": {"age_sec": 4.554, "exists": true, "mtime": "2026-07-01T18:53:53.218156+09:00", "mtime_epoch": 1782899633.218156, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189062}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:53:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 889.98, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.375, "pushed": 3.375, "read": 3.342}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.625, "count": 100, "max_ms": 13.424, "p95_ms": 10.939}, "gstreamer_push": {"avg_ms": 0.995, "count": 101, "max_ms": 3.122, "p95_ms": 1.838}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.901, "count": 100, "max_ms": 69.176, "p95_ms": 64.827}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.483, "count": 100, "max_ms": 93.293, "p95_ms": 89.695}, "yolo11_640_fire_11class": {"avg_ms": 55.989, "count": 100, "max_ms": 75.347, "p95_ms": 72.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.939, "count": 100, "max_ms": 92.089, "p95_ms": 90.033}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.617, "count": 101, "max_ms": 318.725, "p95_ms": 273.499}, "read_interval": {"avg_ms": 253.664, "count": 100, "max_ms": 330.194, "p95_ms": 282.804}, "triton_wait": {"avg_ms": 70.143, "count": 100, "max_ms": 93.306, "p95_ms": 90.478}, "visualize": {"avg_ms": 11.308, "count": 100, "max_ms": 16.016, "p95_ms": 12.772}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 507}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4019.543, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b89e0909c0] error while decoding MB 19 13, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 108, "h264_decode_error_count": 111, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:55:46+09:00", "last_error_summary": "[h264 @ 0x55b89e0909c0] error while decoding MB 19 13, bytestream -5", "last_exit_at": "2026-07-01T18:53:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.78, "last_started_at": "2026-07-01T18:53:55+09:00", "pid": 17078, "process": {"exists": false, "pid": 17078}, "restart_count": 125, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 126}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.234, "sample_count": 1168, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60923, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:55:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T18:55:46+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T18:55:46+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T18:55:46.506050+09:00", "mtime_epoch": 1782899746.5060503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192158}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:55:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 882.027, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.05, "pushed": 4.05, "read": 4.05}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.32, "count": 120, "max_ms": 12.426, "p95_ms": 9.986}, "gstreamer_push": {"avg_ms": 0.934, "count": 121, "max_ms": 2.813, "p95_ms": 1.769}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.375, "count": 120, "max_ms": 70.463, "p95_ms": 60.489}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.433, "count": 120, "max_ms": 95.664, "p95_ms": 85.571}, "yolo11_640_fire_11class": {"avg_ms": 52.662, "count": 120, "max_ms": 73.503, "p95_ms": 68.547}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.566, "count": 120, "max_ms": 95.537, "p95_ms": 86.187}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.709, "count": 121, "max_ms": 317.726, "p95_ms": 259.359}, "read_interval": {"avg_ms": 248.408, "count": 121, "max_ms": 328.119, "p95_ms": 268.289}, "triton_wait": {"avg_ms": 67.371, "count": 120, "max_ms": 95.677, "p95_ms": 87.692}, "visualize": {"avg_ms": 11.197, "count": 120, "max_ms": 15.116, "p95_ms": 12.445}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 507}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4024.684, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:55:46+09:00", "last_error_summary": "[h264 @ 0x55b89e0909c0] error while decoding MB 19 13, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 108, "h264_decode_error_count": 111, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:55:46+09:00", "last_error_summary": "[h264 @ 0x55b89e0909c0] error while decoding MB 19 13, bytestream -5", "last_exit_at": "2026-07-01T18:55:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.985, "last_started_at": "2026-07-01T18:53:55+09:00", "pid": 17078, "process": {"exists": false, "pid": 17078}, "restart_count": 125, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 126}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.234, "sample_count": 1168, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60924, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:55:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:55:46+09:00", "last_frame_read_age_sec": 0.26, "last_frame_read_at": "2026-07-01T18:55:46+09:00", "mjpeg": {"age_sec": 0.25, "exists": true, "mtime": "2026-07-01T18:55:46.506050+09:00", "mtime_epoch": 1782899746.5060503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192158}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:55:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 887.953, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.054, "pushed": 4.054, "read": 4.021}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.32, "count": 121, "max_ms": 12.426, "p95_ms": 9.986}, "gstreamer_push": {"avg_ms": 0.918, "count": 121, "max_ms": 1.889, "p95_ms": 1.765}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.471, "count": 121, "max_ms": 70.463, "p95_ms": 60.489}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.568, "count": 121, "max_ms": 95.664, "p95_ms": 85.571}, "yolo11_640_fire_11class": {"avg_ms": 52.795, "count": 121, "max_ms": 73.503, "p95_ms": 68.678}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.741, "count": 121, "max_ms": 95.537, "p95_ms": 87.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.84, "count": 121, "max_ms": 317.726, "p95_ms": 259.359}, "read_interval": {"avg_ms": 248.393, "count": 120, "max_ms": 328.119, "p95_ms": 268.289}, "triton_wait": {"avg_ms": 67.54, "count": 121, "max_ms": 95.677, "p95_ms": 87.811}, "visualize": {"avg_ms": 11.204, "count": 121, "max_ms": 15.116, "p95_ms": 12.445}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.844, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 507}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4024.676, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 108, "h264_decode_error_count": 111, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:55:46+09:00", "last_error_summary": "[h264 @ 0x55b89e0909c0] error while decoding MB 19 13, bytestream -5", "last_exit_at": "2026-07-01T18:55:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.989, "last_started_at": "2026-07-01T18:53:55+09:00", "pid": 17078, "process": {"exists": false, "pid": 17078}, "restart_count": 125, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 126}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.234, "sample_count": 1168, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60924, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:55:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:55:46+09:00", "last_frame_read_age_sec": 0.263, "last_frame_read_at": "2026-07-01T18:55:46+09:00", "mjpeg": {"age_sec": 0.253, "exists": true, "mtime": "2026-07-01T18:55:46.506050+09:00", "mtime_epoch": 1782899746.5060503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192158}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:55:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 887.953, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.054, "pushed": 4.054, "read": 4.02}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.32, "count": 121, "max_ms": 12.426, "p95_ms": 9.986}, "gstreamer_push": {"avg_ms": 0.918, "count": 121, "max_ms": 1.889, "p95_ms": 1.765}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.471, "count": 121, "max_ms": 70.463, "p95_ms": 60.489}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.568, "count": 121, "max_ms": 95.664, "p95_ms": 85.571}, "yolo11_640_fire_11class": {"avg_ms": 52.795, "count": 121, "max_ms": 73.503, "p95_ms": 68.678}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.741, "count": 121, "max_ms": 95.537, "p95_ms": 87.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.84, "count": 121, "max_ms": 317.726, "p95_ms": 259.359}, "read_interval": {"avg_ms": 248.393, "count": 120, "max_ms": 328.119, "p95_ms": 268.289}, "triton_wait": {"avg_ms": 67.54, "count": 121, "max_ms": 95.677, "p95_ms": 87.811}, "visualize": {"avg_ms": 11.204, "count": 121, "max_ms": 15.116, "p95_ms": 12.445}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.847, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 507}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4132.684, "limit_mb": 8796093022207.996}, "detail": {"pid": 17185, "restart_count": 126, "start_count": 127}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 108, "h264_decode_error_count": 111, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:55:46+09:00", "last_error_summary": "[h264 @ 0x55b89e0909c0] error while decoding MB 19 13, bytestream -5", "last_exit_at": "2026-07-01T18:55:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:55:49+09:00", "pid": 17185, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.227, "name": "ffmpeg", "pid": 17185, "rss_mb": 139.227, "state": "S (sleeping)", "threads": 106}, "restart_count": 126, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 127}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.447, "sample_count": 1149, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 60924, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:55:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.641, "last_frame_pushed_at": "2026-07-01T18:55:46+09:00", "last_frame_read_age_sec": 4.899, "last_frame_read_at": "2026-07-01T18:55:46+09:00", "mjpeg": {"age_sec": 4.625, "exists": true, "mtime": "2026-07-01T18:55:46.770050+09:00", "mtime_epoch": 1782899746.7700496, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192183}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:55:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 887.953, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.435, "pushed": 3.435, "read": 3.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.358, "count": 102, "max_ms": 12.426, "p95_ms": 10.016}, "gstreamer_push": {"avg_ms": 0.91, "count": 103, "max_ms": 1.889, "p95_ms": 1.343}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.927, "count": 102, "max_ms": 70.463, "p95_ms": 59.725}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.372, "count": 102, "max_ms": 95.664, "p95_ms": 82.668}, "yolo11_640_fire_11class": {"avg_ms": 52.694, "count": 102, "max_ms": 73.503, "p95_ms": 67.579}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.47, "count": 102, "max_ms": 94.653, "p95_ms": 84.433}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.856, "count": 103, "max_ms": 317.726, "p95_ms": 258.534}, "read_interval": {"avg_ms": 248.393, "count": 102, "max_ms": 328.119, "p95_ms": 267.699}, "triton_wait": {"avg_ms": 67.308, "count": 102, "max_ms": 95.677, "p95_ms": 86.292}, "visualize": {"avg_ms": 11.263, "count": 102, "max_ms": 15.116, "p95_ms": 12.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 511}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4015.414, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "last_exit_at": "2026-07-01T18:55:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.974, "last_started_at": "2026-07-01T18:55:49+09:00", "pid": 17185, "process": {"exists": false, "pid": 17185}, "restart_count": 126, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 127}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.256, "sample_count": 1169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 61360, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:57:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.045, "last_frame_pushed_at": "2026-07-01T18:57:39+09:00", "last_frame_read_age_sec": 0.037, "last_frame_read_at": "2026-07-01T18:57:39+09:00", "mjpeg": {"age_sec": 0.026, "exists": true, "mtime": "2026-07-01T18:57:39.341978+09:00", "mtime_epoch": 1782899859.3419783, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190563}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:57:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 876.441, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.129, "pushed": 4.129, "read": 4.129}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.293, "count": 123, "max_ms": 11.704, "p95_ms": 10.433}, "gstreamer_push": {"avg_ms": 0.929, "count": 123, "max_ms": 2.925, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.791, "count": 123, "max_ms": 71.106, "p95_ms": 61.162}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.902, "count": 123, "max_ms": 100.518, "p95_ms": 87.75}, "yolo11_640_fire_11class": {"avg_ms": 56.774, "count": 123, "max_ms": 81.152, "p95_ms": 72.139}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.486, "count": 123, "max_ms": 99.09, "p95_ms": 87.342}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.078, "count": 123, "max_ms": 287.36, "p95_ms": 258.502}, "read_interval": {"avg_ms": 243.701, "count": 123, "max_ms": 300.175, "p95_ms": 268.996}, "triton_wait": {"avg_ms": 68.476, "count": 123, "max_ms": 100.529, "p95_ms": 88.149}, "visualize": {"avg_ms": 10.944, "count": 123, "max_ms": 12.604, "p95_ms": 12.021}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.786, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 511}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4025.863, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "last_exit_at": "2026-07-01T18:57:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.243, "last_started_at": "2026-07-01T18:55:49+09:00", "pid": 17185, "process": {"exists": false, "pid": 17185}, "restart_count": 126, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 127}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.447, "sample_count": 1169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 61361, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:57:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:57:39+09:00", "last_frame_read_age_sec": 0.306, "last_frame_read_at": "2026-07-01T18:57:39+09:00", "mjpeg": {"age_sec": 0.295, "exists": true, "mtime": "2026-07-01T18:57:39.341978+09:00", "mtime_epoch": 1782899859.3419783, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190563}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:57:39+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 886.977, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.125, "pushed": 4.125, "read": 4.092}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.292, "count": 123, "max_ms": 11.704, "p95_ms": 10.433}, "gstreamer_push": {"avg_ms": 0.946, "count": 123, "max_ms": 3.016, "p95_ms": 0.941}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.866, "count": 123, "max_ms": 71.106, "p95_ms": 61.162}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.896, "count": 123, "max_ms": 100.518, "p95_ms": 87.75}, "yolo11_640_fire_11class": {"avg_ms": 56.79, "count": 123, "max_ms": 81.152, "p95_ms": 72.139}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.51, "count": 123, "max_ms": 99.09, "p95_ms": 87.342}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.668, "count": 123, "max_ms": 299.347, "p95_ms": 260.08}, "read_interval": {"avg_ms": 243.755, "count": 122, "max_ms": 300.175, "p95_ms": 268.996}, "triton_wait": {"avg_ms": 68.476, "count": 123, "max_ms": 100.529, "p95_ms": 88.149}, "visualize": {"avg_ms": 10.948, "count": 123, "max_ms": 12.604, "p95_ms": 12.021}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.816, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 511}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4025.855, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "last_exit_at": "2026-07-01T18:57:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.246, "last_started_at": "2026-07-01T18:55:49+09:00", "pid": 17185, "process": {"exists": false, "pid": 17185}, "restart_count": 126, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 127}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.447, "sample_count": 1169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 61361, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:57:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:57:39+09:00", "last_frame_read_age_sec": 0.309, "last_frame_read_at": "2026-07-01T18:57:39+09:00", "mjpeg": {"age_sec": 0.298, "exists": true, "mtime": "2026-07-01T18:57:39.341978+09:00", "mtime_epoch": 1782899859.3419783, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190563}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:57:39+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 886.977, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.125, "pushed": 4.125, "read": 4.091}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.292, "count": 123, "max_ms": 11.704, "p95_ms": 10.433}, "gstreamer_push": {"avg_ms": 0.946, "count": 123, "max_ms": 3.016, "p95_ms": 0.941}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.866, "count": 123, "max_ms": 71.106, "p95_ms": 61.162}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.896, "count": 123, "max_ms": 100.518, "p95_ms": 87.75}, "yolo11_640_fire_11class": {"avg_ms": 56.79, "count": 123, "max_ms": 81.152, "p95_ms": 72.139}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.51, "count": 123, "max_ms": 99.09, "p95_ms": 87.342}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.668, "count": 123, "max_ms": 299.347, "p95_ms": 260.08}, "read_interval": {"avg_ms": 243.755, "count": 122, "max_ms": 300.175, "p95_ms": 268.996}, "triton_wait": {"avg_ms": 68.476, "count": 123, "max_ms": 100.529, "p95_ms": 88.149}, "visualize": {"avg_ms": 10.948, "count": 123, "max_ms": 12.604, "p95_ms": 12.021}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.819, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 511}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4132.824, "limit_mb": 8796093022207.996}, "detail": {"pid": 17292, "restart_count": 127, "start_count": 128}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "last_exit_at": "2026-07-01T18:57:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:57:42+09:00", "pid": 17292, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.031, "name": "ffmpeg", "pid": 17292, "rss_mb": 139.031, "state": "S (sleeping)", "threads": 106}, "restart_count": 127, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 128}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.499, "sample_count": 1150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 61361, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:57:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.452, "last_frame_pushed_at": "2026-07-01T18:57:39+09:00", "last_frame_read_age_sec": 4.755, "last_frame_read_at": "2026-07-01T18:57:39+09:00", "mjpeg": {"age_sec": 4.436, "exists": true, "mtime": "2026-07-01T18:57:39.649979+09:00", "mtime_epoch": 1782899859.6499786, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190714}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:57:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 886.977, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.516, "pushed": 3.516, "read": 3.482}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.281, "count": 104, "max_ms": 11.704, "p95_ms": 10.433}, "gstreamer_push": {"avg_ms": 0.951, "count": 105, "max_ms": 3.016, "p95_ms": 0.941}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.839, "count": 104, "max_ms": 71.106, "p95_ms": 61.264}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.495, "count": 104, "max_ms": 100.518, "p95_ms": 87.51}, "yolo11_640_fire_11class": {"avg_ms": 56.608, "count": 104, "max_ms": 80.328, "p95_ms": 70.959}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.463, "count": 104, "max_ms": 99.09, "p95_ms": 86.476}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.567, "count": 105, "max_ms": 299.347, "p95_ms": 261.624}, "read_interval": {"avg_ms": 243.56, "count": 104, "max_ms": 300.175, "p95_ms": 269.626}, "triton_wait": {"avg_ms": 68.178, "count": 104, "max_ms": 100.529, "p95_ms": 87.762}, "visualize": {"avg_ms": 10.953, "count": 104, "max_ms": 12.604, "p95_ms": 12.12}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.865, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 514}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4032.207, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "last_exit_at": "2026-07-01T18:59:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.454, "last_started_at": "2026-07-01T18:57:42+09:00", "pid": 17292, "process": {"exists": false, "pid": 17292}, "restart_count": 127, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 128}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.457, "sample_count": 1180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 61796, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:59:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T18:59:30+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T18:59:30+09:00", "mjpeg": {"age_sec": 0.221, "exists": true, "mtime": "2026-07-01T18:59:30.318251+09:00", "mtime_epoch": 1782899970.318251, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185280}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:59:30+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 893.512, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.174, "pushed": 4.174, "read": 4.14}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.235, "count": 124, "max_ms": 10.832, "p95_ms": 10.512}, "gstreamer_push": {"avg_ms": 0.907, "count": 125, "max_ms": 3.016, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.82, "count": 124, "max_ms": 71.487, "p95_ms": 59.447}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.008, "count": 124, "max_ms": 92.822, "p95_ms": 89.713}, "yolo11_640_fire_11class": {"avg_ms": 55.193, "count": 124, "max_ms": 74.28, "p95_ms": 71.254}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.305, "count": 124, "max_ms": 91.672, "p95_ms": 88.363}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.986, "count": 125, "max_ms": 323.243, "p95_ms": 266.791}, "read_interval": {"avg_ms": 241.501, "count": 124, "max_ms": 332.282, "p95_ms": 275.963}, "triton_wait": {"avg_ms": 68.432, "count": 124, "max_ms": 92.834, "p95_ms": 90.589}, "visualize": {"avg_ms": 10.769, "count": 124, "max_ms": 12.451, "p95_ms": 12.201}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 514}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4032.199, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "last_exit_at": "2026-07-01T18:59:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.457, "last_started_at": "2026-07-01T18:57:42+09:00", "pid": 17292, "process": {"exists": false, "pid": 17292}, "restart_count": 127, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 128}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.457, "sample_count": 1179, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 61796, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:59:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T18:59:30+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-01T18:59:30+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T18:59:30.318251+09:00", "mtime_epoch": 1782899970.318251, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185280}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:59:30+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 893.512, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.173, "pushed": 4.173, "read": 4.14}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.235, "count": 124, "max_ms": 10.832, "p95_ms": 10.512}, "gstreamer_push": {"avg_ms": 0.907, "count": 125, "max_ms": 3.016, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.82, "count": 124, "max_ms": 71.487, "p95_ms": 59.447}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.008, "count": 124, "max_ms": 92.822, "p95_ms": 89.713}, "yolo11_640_fire_11class": {"avg_ms": 55.193, "count": 124, "max_ms": 74.28, "p95_ms": 71.254}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.305, "count": 124, "max_ms": 91.672, "p95_ms": 88.363}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.986, "count": 125, "max_ms": 323.243, "p95_ms": 266.791}, "read_interval": {"avg_ms": 241.501, "count": 124, "max_ms": 332.282, "p95_ms": 275.963}, "triton_wait": {"avg_ms": 68.432, "count": 124, "max_ms": 92.834, "p95_ms": 90.589}, "visualize": {"avg_ms": 10.769, "count": 124, "max_ms": 12.451, "p95_ms": 12.201}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 514}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4139.172, "limit_mb": 8796093022207.996}, "detail": {"pid": 17399, "restart_count": 128, "start_count": 129}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "last_exit_at": "2026-07-01T18:59:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T18:59:32+09:00", "pid": 17399, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.375, "name": "ffmpeg", "pid": 17399, "rss_mb": 139.375, "state": "S (sleeping)", "threads": 106}, "restart_count": 128, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 129}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.509, "sample_count": 1163, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 61796, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T18:59:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.099, "last_frame_pushed_at": "2026-07-01T18:59:30+09:00", "last_frame_read_age_sec": 4.326, "last_frame_read_at": "2026-07-01T18:59:30+09:00", "mjpeg": {"age_sec": 4.081, "exists": true, "mtime": "2026-07-01T18:59:30.554252+09:00", "mtime_epoch": 1782899970.554252, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186127}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T18:59:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 893.512, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.593, "pushed": 3.593, "read": 3.56}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.266, "count": 107, "max_ms": 10.832, "p95_ms": 10.512}, "gstreamer_push": {"avg_ms": 0.909, "count": 107, "max_ms": 3.016, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.829, "count": 107, "max_ms": 71.487, "p95_ms": 59.447}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.21, "count": 107, "max_ms": 92.822, "p95_ms": 90.576}, "yolo11_640_fire_11class": {"avg_ms": 55.182, "count": 107, "max_ms": 74.28, "p95_ms": 71.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.639, "count": 107, "max_ms": 91.672, "p95_ms": 89.076}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.694, "count": 107, "max_ms": 323.243, "p95_ms": 267.254}, "read_interval": {"avg_ms": 242.231, "count": 106, "max_ms": 332.282, "p95_ms": 276.682}, "triton_wait": {"avg_ms": 68.679, "count": 107, "max_ms": 92.834, "p95_ms": 91.026}, "visualize": {"avg_ms": 10.763, "count": 107, "max_ms": 12.451, "p95_ms": 12.204}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.779, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 518}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4024.867, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "last_exit_at": "2026-07-01T19:01:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.597, "last_started_at": "2026-07-01T18:59:32+09:00", "pid": 17399, "process": {"exists": false, "pid": 17399}, "restart_count": 128, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 129}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.443, "sample_count": 1189, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 62237, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:01:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T19:01:22+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T19:01:21+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T19:01:22.006802+09:00", "mtime_epoch": 1782900082.0068016, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:01:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 885.613, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.082, "pushed": 4.082, "read": 4.048}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.509, "count": 121, "max_ms": 14.849, "p95_ms": 12.015}, "gstreamer_push": {"avg_ms": 0.975, "count": 122, "max_ms": 2.978, "p95_ms": 1.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.257, "count": 121, "max_ms": 69.779, "p95_ms": 63.95}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.716, "count": 121, "max_ms": 92.972, "p95_ms": 86.501}, "yolo11_640_fire_11class": {"avg_ms": 54.178, "count": 121, "max_ms": 73.816, "p95_ms": 68.834}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.597, "count": 121, "max_ms": 96.571, "p95_ms": 88.34}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.702, "count": 122, "max_ms": 322.756, "p95_ms": 267.62}, "read_interval": {"avg_ms": 246.976, "count": 121, "max_ms": 331.987, "p95_ms": 276.079}, "triton_wait": {"avg_ms": 69.581, "count": 121, "max_ms": 96.582, "p95_ms": 89.454}, "visualize": {"avg_ms": 11.45, "count": 121, "max_ms": 15.933, "p95_ms": 15.098}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 518}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4024.859, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "last_exit_at": "2026-07-01T19:01:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.6, "last_started_at": "2026-07-01T18:59:32+09:00", "pid": 17399, "process": {"exists": false, "pid": 17399}, "restart_count": 128, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 129}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.443, "sample_count": 1189, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 62237, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:01:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T19:01:22+09:00", "last_frame_read_age_sec": 0.241, "last_frame_read_at": "2026-07-01T19:01:21+09:00", "mjpeg": {"age_sec": 0.227, "exists": true, "mtime": "2026-07-01T19:01:22.006802+09:00", "mtime_epoch": 1782900082.0068016, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:01:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 885.613, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.081, "pushed": 4.081, "read": 4.048}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.509, "count": 121, "max_ms": 14.849, "p95_ms": 12.015}, "gstreamer_push": {"avg_ms": 0.975, "count": 122, "max_ms": 2.978, "p95_ms": 1.889}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.257, "count": 121, "max_ms": 69.779, "p95_ms": 63.95}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.716, "count": 121, "max_ms": 92.972, "p95_ms": 86.501}, "yolo11_640_fire_11class": {"avg_ms": 54.178, "count": 121, "max_ms": 73.816, "p95_ms": 68.834}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.597, "count": 121, "max_ms": 96.571, "p95_ms": 88.34}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.702, "count": 122, "max_ms": 322.756, "p95_ms": 267.62}, "read_interval": {"avg_ms": 246.976, "count": 121, "max_ms": 331.987, "p95_ms": 276.079}, "triton_wait": {"avg_ms": 69.581, "count": 121, "max_ms": 96.582, "p95_ms": 89.454}, "visualize": {"avg_ms": 11.45, "count": 121, "max_ms": 15.933, "p95_ms": 15.098}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 518}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4136.023, "limit_mb": 8796093022207.996}, "detail": {"pid": 17506, "restart_count": 129, "start_count": 130}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 109, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T18:57:39+09:00", "last_error_summary": "[h264 @ 0x5635c93b5180] error while decoding MB 70 39, bytestream -5", "last_exit_at": "2026-07-01T19:01:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:01:24+09:00", "pid": 17506, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.973, "name": "ffmpeg", "pid": 17506, "rss_mb": 138.973, "state": "S (sleeping)", "threads": 106}, "restart_count": 129, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 130}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.597, "sample_count": 1171, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 62237, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:01:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.306, "last_frame_pushed_at": "2026-07-01T19:01:22+09:00", "last_frame_read_age_sec": 4.539, "last_frame_read_at": "2026-07-01T19:01:21+09:00", "mjpeg": {"age_sec": 4.29, "exists": true, "mtime": "2026-07-01T19:01:22.242803+09:00", "mtime_epoch": 1782900082.242803, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:01:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 885.613, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.483, "pushed": 3.483, "read": 3.45}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.624, "count": 103, "max_ms": 14.849, "p95_ms": 12.287}, "gstreamer_push": {"avg_ms": 0.983, "count": 104, "max_ms": 2.978, "p95_ms": 1.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.556, "count": 103, "max_ms": 69.779, "p95_ms": 64.37}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.96, "count": 103, "max_ms": 92.972, "p95_ms": 86.501}, "yolo11_640_fire_11class": {"avg_ms": 54.524, "count": 103, "max_ms": 73.816, "p95_ms": 69.652}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.145, "count": 103, "max_ms": 96.571, "p95_ms": 88.364}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.848, "count": 104, "max_ms": 322.756, "p95_ms": 268.443}, "read_interval": {"avg_ms": 248.065, "count": 103, "max_ms": 331.987, "p95_ms": 276.761}, "triton_wait": {"avg_ms": 69.946, "count": 103, "max_ms": 96.582, "p95_ms": 89.454}, "visualize": {"avg_ms": 11.436, "count": 103, "max_ms": 15.933, "p95_ms": 14.879}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.856, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 522}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4032.047, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5597ac6c1640] error while decoding MB 58 28, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 110, "h264_decode_error_count": 113, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:03:14+09:00", "last_error_summary": "[h264 @ 0x5597ac6c1640] error while decoding MB 58 28, bytestream -6", "last_exit_at": "2026-07-01T19:01:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.633, "last_started_at": "2026-07-01T19:01:24+09:00", "pid": 17506, "process": {"exists": false, "pid": 17506}, "restart_count": 129, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 130}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.399, "sample_count": 1184, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 62669, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:03:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T19:03:14+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T19:03:14+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T19:03:14.123579+09:00", "mtime_epoch": 1782900194.1235788, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191191}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:03:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 888.273, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.993, "pushed": 3.993, "read": 3.993}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.87, "count": 119, "max_ms": 15.65, "p95_ms": 13.711}, "gstreamer_push": {"avg_ms": 0.951, "count": 119, "max_ms": 2.922, "p95_ms": 1.79}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.293, "count": 119, "max_ms": 70.544, "p95_ms": 61.699}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.028, "count": 119, "max_ms": 93.959, "p95_ms": 88.901}, "yolo11_640_fire_11class": {"avg_ms": 52.075, "count": 119, "max_ms": 70.679, "p95_ms": 69.16}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.551, "count": 119, "max_ms": 93.098, "p95_ms": 90.911}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.201, "count": 119, "max_ms": 340.007, "p95_ms": 277.795}, "read_interval": {"avg_ms": 251.881, "count": 119, "max_ms": 349.864, "p95_ms": 286.969}, "triton_wait": {"avg_ms": 68.715, "count": 119, "max_ms": 93.971, "p95_ms": 91.137}, "visualize": {"avg_ms": 11.245, "count": 119, "max_ms": 16.699, "p95_ms": 12.402}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.802, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 522}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4031.148, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:03:14+09:00", "last_error_summary": "[h264 @ 0x5597ac6c1640] error while decoding MB 58 28, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 110, "h264_decode_error_count": 113, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:03:14+09:00", "last_error_summary": "[h264 @ 0x5597ac6c1640] error while decoding MB 58 28, bytestream -6", "last_exit_at": "2026-07-01T19:03:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.81, "last_started_at": "2026-07-01T19:01:24+09:00", "pid": 17506, "process": {"exists": false, "pid": 17506}, "restart_count": 129, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 130}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.399, "sample_count": 1184, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 62670, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:03:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:03:14+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T19:03:14+09:00", "mjpeg": {"age_sec": 0.218, "exists": true, "mtime": "2026-07-01T19:03:14.123579+09:00", "mtime_epoch": 1782900194.1235788, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191191}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:03:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 888.273, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.003, "pushed": 4.003, "read": 3.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.873, "count": 119, "max_ms": 15.65, "p95_ms": 13.711}, "gstreamer_push": {"avg_ms": 0.95, "count": 120, "max_ms": 2.922, "p95_ms": 1.789}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.249, "count": 119, "max_ms": 70.544, "p95_ms": 61.699}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.044, "count": 119, "max_ms": 93.959, "p95_ms": 88.901}, "yolo11_640_fire_11class": {"avg_ms": 52.063, "count": 119, "max_ms": 70.679, "p95_ms": 69.16}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.553, "count": 119, "max_ms": 93.098, "p95_ms": 90.911}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.063, "count": 120, "max_ms": 340.007, "p95_ms": 274.743}, "read_interval": {"avg_ms": 251.881, "count": 119, "max_ms": 349.864, "p95_ms": 286.969}, "triton_wait": {"avg_ms": 68.724, "count": 119, "max_ms": 93.971, "p95_ms": 91.137}, "visualize": {"avg_ms": 11.247, "count": 119, "max_ms": 16.699, "p95_ms": 12.402}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 522}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4031.141, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 110, "h264_decode_error_count": 113, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:03:14+09:00", "last_error_summary": "[h264 @ 0x5597ac6c1640] error while decoding MB 58 28, bytestream -6", "last_exit_at": "2026-07-01T19:03:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.814, "last_started_at": "2026-07-01T19:01:24+09:00", "pid": 17506, "process": {"exists": false, "pid": 17506}, "restart_count": 129, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 130}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.399, "sample_count": 1184, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 62670, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:03:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:03:14+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-01T19:03:14+09:00", "mjpeg": {"age_sec": 0.221, "exists": true, "mtime": "2026-07-01T19:03:14.123579+09:00", "mtime_epoch": 1782900194.1235788, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191191}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:03:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 888.273, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.002, "pushed": 4.002, "read": 3.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.873, "count": 119, "max_ms": 15.65, "p95_ms": 13.711}, "gstreamer_push": {"avg_ms": 0.95, "count": 120, "max_ms": 2.922, "p95_ms": 1.789}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.249, "count": 119, "max_ms": 70.544, "p95_ms": 61.699}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.044, "count": 119, "max_ms": 93.959, "p95_ms": 88.901}, "yolo11_640_fire_11class": {"avg_ms": 52.063, "count": 119, "max_ms": 70.679, "p95_ms": 69.16}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.553, "count": 119, "max_ms": 93.098, "p95_ms": 90.911}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.063, "count": 120, "max_ms": 340.007, "p95_ms": 274.743}, "read_interval": {"avg_ms": 251.881, "count": 119, "max_ms": 349.864, "p95_ms": 286.969}, "triton_wait": {"avg_ms": 68.724, "count": 119, "max_ms": 93.971, "p95_ms": 91.137}, "visualize": {"avg_ms": 11.247, "count": 119, "max_ms": 16.699, "p95_ms": 12.402}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 522}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4136.305, "limit_mb": 8796093022207.996}, "detail": {"pid": 17613, "restart_count": 130, "start_count": 131}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 110, "h264_decode_error_count": 113, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:03:14+09:00", "last_error_summary": "[h264 @ 0x5597ac6c1640] error while decoding MB 58 28, bytestream -6", "last_exit_at": "2026-07-01T19:03:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:03:17+09:00", "pid": 17613, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.688, "name": "ffmpeg", "pid": 17613, "rss_mb": 136.688, "state": "S (sleeping)", "threads": 106}, "restart_count": 130, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 131}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.399, "sample_count": 1165, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 62670, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:03:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.821, "last_frame_pushed_at": "2026-07-01T19:03:14+09:00", "last_frame_read_age_sec": 5.048, "last_frame_read_at": "2026-07-01T19:03:14+09:00", "mjpeg": {"age_sec": 4.804, "exists": true, "mtime": "2026-07-01T19:03:14.355581+09:00", "mtime_epoch": 1782900194.3555806, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191186}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:03:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 888.273, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.35, "pushed": 3.35, "read": 3.316}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.008, "count": 100, "max_ms": 15.65, "p95_ms": 13.711}, "gstreamer_push": {"avg_ms": 0.951, "count": 100, "max_ms": 2.922, "p95_ms": 1.789}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.502, "count": 100, "max_ms": 70.544, "p95_ms": 61.558}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.041, "count": 100, "max_ms": 93.959, "p95_ms": 88.576}, "yolo11_640_fire_11class": {"avg_ms": 51.223, "count": 100, "max_ms": 70.679, "p95_ms": 69.16}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.362, "count": 100, "max_ms": 93.098, "p95_ms": 90.911}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.982, "count": 100, "max_ms": 340.007, "p95_ms": 277.795}, "read_interval": {"avg_ms": 252.837, "count": 99, "max_ms": 349.864, "p95_ms": 287.98}, "triton_wait": {"avg_ms": 67.681, "count": 100, "max_ms": 93.971, "p95_ms": 91.137}, "visualize": {"avg_ms": 11.304, "count": 100, "max_ms": 16.699, "p95_ms": 12.402}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.852, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 525}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4030.113, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f27e584600] error while decoding MB 27 24, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 111, "h264_decode_error_count": 114, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:05:11+09:00", "last_error_summary": "[h264 @ 0x55f27e584600] error while decoding MB 27 24, bytestream -9", "last_exit_at": "2026-07-01T19:03:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.024, "last_started_at": "2026-07-01T19:03:17+09:00", "pid": 17613, "process": {"exists": false, "pid": 17613}, "restart_count": 130, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 131}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.414, "sample_count": 1170, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63116, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:05:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T19:05:11+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T19:05:11+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T19:05:11.140581+09:00", "mtime_epoch": 1782900311.1405814, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188066}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:05:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 889.137, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.99, "pushed": 3.99, "read": 3.99}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.613, "count": 119, "max_ms": 11.815, "p95_ms": 10.55}, "gstreamer_push": {"avg_ms": 0.94, "count": 119, "max_ms": 2.862, "p95_ms": 1.861}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.771, "count": 119, "max_ms": 69.428, "p95_ms": 61.381}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.21, "count": 119, "max_ms": 93.732, "p95_ms": 90.177}, "yolo11_640_fire_11class": {"avg_ms": 53.593, "count": 119, "max_ms": 74.077, "p95_ms": 71.225}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.476, "count": 119, "max_ms": 92.775, "p95_ms": 89.983}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.362, "count": 119, "max_ms": 344.613, "p95_ms": 283.575}, "read_interval": {"avg_ms": 252.081, "count": 119, "max_ms": 353.847, "p95_ms": 292.764}, "triton_wait": {"avg_ms": 68.47, "count": 119, "max_ms": 93.745, "p95_ms": 90.189}, "visualize": {"avg_ms": 11.467, "count": 119, "max_ms": 16.192, "p95_ms": 14.108}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.823, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 525}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4029.461, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:05:11+09:00", "last_error_summary": "[h264 @ 0x55f27e584600] error while decoding MB 27 24, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 111, "h264_decode_error_count": 114, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:05:11+09:00", "last_error_summary": "[h264 @ 0x55f27e584600] error while decoding MB 27 24, bytestream -9", "last_exit_at": "2026-07-01T19:05:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.21, "last_started_at": "2026-07-01T19:03:17+09:00", "pid": 17613, "process": {"exists": false, "pid": 17613}, "restart_count": 130, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 131}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.414, "sample_count": 1170, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63117, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:05:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T19:05:11+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T19:05:11+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T19:05:11.140581+09:00", "mtime_epoch": 1782900311.1405814, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188066}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:05:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 889.137, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.967, "pushed": 3.967, "read": 3.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.616, "count": 119, "max_ms": 11.815, "p95_ms": 10.55}, "gstreamer_push": {"avg_ms": 0.941, "count": 119, "max_ms": 2.862, "p95_ms": 1.861}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.728, "count": 119, "max_ms": 69.428, "p95_ms": 61.381}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.232, "count": 119, "max_ms": 93.732, "p95_ms": 90.177}, "yolo11_640_fire_11class": {"avg_ms": 53.618, "count": 119, "max_ms": 74.077, "p95_ms": 71.225}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.52, "count": 119, "max_ms": 92.775, "p95_ms": 89.983}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.403, "count": 119, "max_ms": 344.613, "p95_ms": 283.575}, "read_interval": {"avg_ms": 252.081, "count": 119, "max_ms": 353.847, "p95_ms": 292.764}, "triton_wait": {"avg_ms": 68.499, "count": 119, "max_ms": 93.745, "p95_ms": 90.189}, "visualize": {"avg_ms": 11.471, "count": 119, "max_ms": 16.192, "p95_ms": 14.108}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 525}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4029.453, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 111, "h264_decode_error_count": 114, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:05:11+09:00", "last_error_summary": "[h264 @ 0x55f27e584600] error while decoding MB 27 24, bytestream -9", "last_exit_at": "2026-07-01T19:05:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.213, "last_started_at": "2026-07-01T19:03:17+09:00", "pid": 17613, "process": {"exists": false, "pid": 17613}, "restart_count": 130, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 131}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.414, "sample_count": 1170, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63117, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:05:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T19:05:11+09:00", "last_frame_read_age_sec": 0.241, "last_frame_read_at": "2026-07-01T19:05:11+09:00", "mjpeg": {"age_sec": 0.231, "exists": true, "mtime": "2026-07-01T19:05:11.140581+09:00", "mtime_epoch": 1782900311.1405814, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188066}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:05:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 889.137, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.967, "pushed": 3.967, "read": 3.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.616, "count": 119, "max_ms": 11.815, "p95_ms": 10.55}, "gstreamer_push": {"avg_ms": 0.941, "count": 119, "max_ms": 2.862, "p95_ms": 1.861}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.728, "count": 119, "max_ms": 69.428, "p95_ms": 61.381}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.232, "count": 119, "max_ms": 93.732, "p95_ms": 90.177}, "yolo11_640_fire_11class": {"avg_ms": 53.618, "count": 119, "max_ms": 74.077, "p95_ms": 71.225}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.52, "count": 119, "max_ms": 92.775, "p95_ms": 89.983}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.403, "count": 119, "max_ms": 344.613, "p95_ms": 283.575}, "read_interval": {"avg_ms": 252.081, "count": 119, "max_ms": 353.847, "p95_ms": 292.764}, "triton_wait": {"avg_ms": 68.499, "count": 119, "max_ms": 93.745, "p95_ms": 90.189}, "visualize": {"avg_ms": 11.471, "count": 119, "max_ms": 16.192, "p95_ms": 14.108}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 525}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4139.457, "limit_mb": 8796093022207.996}, "detail": {"pid": 17720, "restart_count": 131, "start_count": 132}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 111, "h264_decode_error_count": 114, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:05:11+09:00", "last_error_summary": "[h264 @ 0x55f27e584600] error while decoding MB 27 24, bytestream -9", "last_exit_at": "2026-07-01T19:05:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:05:13+09:00", "pid": 17720, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.074, "name": "ffmpeg", "pid": 17720, "rss_mb": 139.074, "state": "S (sleeping)", "threads": 106}, "restart_count": 131, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 132}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.234, "sample_count": 1151, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63117, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:05:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.618, "last_frame_pushed_at": "2026-07-01T19:05:11+09:00", "last_frame_read_age_sec": 4.851, "last_frame_read_at": "2026-07-01T19:05:11+09:00", "mjpeg": {"age_sec": 4.602, "exists": true, "mtime": "2026-07-01T19:05:11.380584+09:00", "mtime_epoch": 1782900311.3805838, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187232}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:05:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 889.137, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.403, "pushed": 3.403, "read": 3.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.694, "count": 101, "max_ms": 11.815, "p95_ms": 10.55}, "gstreamer_push": {"avg_ms": 0.918, "count": 102, "max_ms": 2.841, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.507, "count": 101, "max_ms": 69.428, "p95_ms": 59.936}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.997, "count": 101, "max_ms": 93.732, "p95_ms": 89.776}, "yolo11_640_fire_11class": {"avg_ms": 53.433, "count": 101, "max_ms": 74.077, "p95_ms": 70.437}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.33, "count": 101, "max_ms": 92.775, "p95_ms": 89.171}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.418, "count": 102, "max_ms": 342.873, "p95_ms": 276.783}, "read_interval": {"avg_ms": 252.166, "count": 101, "max_ms": 352.012, "p95_ms": 285.838}, "triton_wait": {"avg_ms": 68.272, "count": 101, "max_ms": 93.745, "p95_ms": 89.785}, "visualize": {"avg_ms": 11.532, "count": 101, "max_ms": 16.192, "p95_ms": 14.108}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 529}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4032.57, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:05:11+09:00", "last_error_summary": "[h264 @ 0x55f27e584600] error while decoding MB 27 24, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 111, "h264_decode_error_count": 114, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:05:11+09:00", "last_error_summary": "[h264 @ 0x55f27e584600] error while decoding MB 27 24, bytestream -9", "last_exit_at": "2026-07-01T19:07:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.322, "last_started_at": "2026-07-01T19:05:13+09:00", "pid": 17720, "process": {"exists": false, "pid": 17720}, "restart_count": 131, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 132}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.64, "sample_count": 1167, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63554, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:07:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:07:04+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T19:07:04+09:00", "mjpeg": {"age_sec": 0.218, "exists": true, "mtime": "2026-07-01T19:07:04.085698+09:00", "mtime_epoch": 1782900424.0856981, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190832}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:07:04+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 891.438, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.94, "pushed": 3.94, "read": 3.907}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.808, "count": 117, "max_ms": 17.732, "p95_ms": 11.37}, "gstreamer_push": {"avg_ms": 0.934, "count": 118, "max_ms": 2.955, "p95_ms": 1.713}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.351, "count": 117, "max_ms": 68.973, "p95_ms": 62.088}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.496, "count": 117, "max_ms": 96.236, "p95_ms": 88.851}, "yolo11_640_fire_11class": {"avg_ms": 53.316, "count": 117, "max_ms": 76.008, "p95_ms": 71.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.929, "count": 117, "max_ms": 95.176, "p95_ms": 88.941}}, "truncated_count": 0}, "processing_total": {"avg_ms": 246.109, "count": 118, "max_ms": 320.344, "p95_ms": 289.032}, "read_interval": {"avg_ms": 255.913, "count": 117, "max_ms": 330.178, "p95_ms": 299.392}, "triton_wait": {"avg_ms": 68.965, "count": 117, "max_ms": 96.248, "p95_ms": 89.413}, "visualize": {"avg_ms": 11.572, "count": 117, "max_ms": 17.477, "p95_ms": 13.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 529}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4032.684, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 111, "h264_decode_error_count": 114, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:05:11+09:00", "last_error_summary": "[h264 @ 0x55f27e584600] error while decoding MB 27 24, bytestream -9", "last_exit_at": "2026-07-01T19:07:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.325, "last_started_at": "2026-07-01T19:05:13+09:00", "pid": 17720, "process": {"exists": false, "pid": 17720}, "restart_count": 131, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 132}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.64, "sample_count": 1167, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63554, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:07:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:07:04+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T19:07:04+09:00", "mjpeg": {"age_sec": 0.221, "exists": true, "mtime": "2026-07-01T19:07:04.085698+09:00", "mtime_epoch": 1782900424.0856981, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190832}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:07:04+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 891.438, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.94, "pushed": 3.94, "read": 3.907}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.808, "count": 117, "max_ms": 17.732, "p95_ms": 11.37}, "gstreamer_push": {"avg_ms": 0.934, "count": 118, "max_ms": 2.955, "p95_ms": 1.713}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.351, "count": 117, "max_ms": 68.973, "p95_ms": 62.088}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.496, "count": 117, "max_ms": 96.236, "p95_ms": 88.851}, "yolo11_640_fire_11class": {"avg_ms": 53.316, "count": 117, "max_ms": 76.008, "p95_ms": 71.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.929, "count": 117, "max_ms": 95.176, "p95_ms": 88.941}}, "truncated_count": 0}, "processing_total": {"avg_ms": 246.109, "count": 118, "max_ms": 320.344, "p95_ms": 289.032}, "read_interval": {"avg_ms": 255.913, "count": 117, "max_ms": 330.178, "p95_ms": 299.392}, "triton_wait": {"avg_ms": 68.965, "count": 117, "max_ms": 96.248, "p95_ms": 89.413}, "visualize": {"avg_ms": 11.572, "count": 117, "max_ms": 17.477, "p95_ms": 13.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 529}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4143.176, "limit_mb": 8796093022207.996}, "detail": {"pid": 17827, "restart_count": 132, "start_count": 133}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 111, "h264_decode_error_count": 114, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:05:11+09:00", "last_error_summary": "[h264 @ 0x55f27e584600] error while decoding MB 27 24, bytestream -9", "last_exit_at": "2026-07-01T19:07:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:07:06+09:00", "pid": 17827, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.305, "name": "ffmpeg", "pid": 17827, "rss_mb": 141.305, "state": "S (sleeping)", "threads": 106}, "restart_count": 132, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 133}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.64, "sample_count": 1148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63554, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:07:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.635, "last_frame_pushed_at": "2026-07-01T19:07:04+09:00", "last_frame_read_age_sec": 4.865, "last_frame_read_at": "2026-07-01T19:07:04+09:00", "mjpeg": {"age_sec": 4.618, "exists": true, "mtime": "2026-07-01T19:07:04.317701+09:00", "mtime_epoch": 1782900424.3177006, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190865}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:07:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 891.438, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.35, "pushed": 3.35, "read": 3.317}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.682, "count": 99, "max_ms": 14.624, "p95_ms": 11.214}, "gstreamer_push": {"avg_ms": 0.944, "count": 100, "max_ms": 2.955, "p95_ms": 1.713}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.68, "count": 99, "max_ms": 68.973, "p95_ms": 62.088}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.128, "count": 99, "max_ms": 96.236, "p95_ms": 89.401}, "yolo11_640_fire_11class": {"avg_ms": 52.689, "count": 99, "max_ms": 76.008, "p95_ms": 71.483}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.328, "count": 99, "max_ms": 95.176, "p95_ms": 88.941}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.035, "count": 100, "max_ms": 320.344, "p95_ms": 289.032}, "read_interval": {"avg_ms": 254.843, "count": 99, "max_ms": 330.178, "p95_ms": 302.062}, "triton_wait": {"avg_ms": 68.511, "count": 99, "max_ms": 96.248, "p95_ms": 89.413}, "visualize": {"avg_ms": 11.474, "count": 99, "max_ms": 16.527, "p95_ms": 13.305}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.847, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 533}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4034.465, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559c510dba00] cabac decode of qscale diff failed at 28 42 [h264 @ 0x559c510dba00] error while decoding MB 28 42, bytestream 2510 [h264 @ 0x559c5117ddc0] error while decoding MB 55 28, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 112, "h264_decode_error_count": 117, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:08:51+09:00", "last_error_summary": "[h264 @ 0x559c510dba00] cabac decode of qscale diff failed at 28 42 [h264 @ 0x559c510dba00] error while decoding MB 28 42, bytestream 2510 [h264 @ 0x559c5117ddc0] error while decoding MB 55 28, bytestream -9", "last_exit_at": "2026-07-01T19:07:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 104.242, "last_started_at": "2026-07-01T19:07:06+09:00", "pid": 17827, "process": {"exists": false, "pid": 17827}, "restart_count": 132, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 133}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.714, "sample_count": 1169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63963, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:08:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T19:08:51+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T19:08:51+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T19:08:51.134865+09:00", "mtime_epoch": 1782900531.1348646, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192382}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:08:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 891.332, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.099, "pushed": 4.099, "read": 4.099}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.14, "count": 122, "max_ms": 12.864, "p95_ms": 10.612}, "gstreamer_push": {"avg_ms": 0.934, "count": 122, "max_ms": 1.884, "p95_ms": 1.798}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.61, "count": 122, "max_ms": 65.929, "p95_ms": 56.623}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.374, "count": 122, "max_ms": 95.745, "p95_ms": 87.747}, "yolo11_640_fire_11class": {"avg_ms": 53.075, "count": 122, "max_ms": 73.454, "p95_ms": 69.455}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.167, "count": 122, "max_ms": 94.864, "p95_ms": 86.559}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.863, "count": 122, "max_ms": 298.172, "p95_ms": 263.434}, "read_interval": {"avg_ms": 245.524, "count": 122, "max_ms": 308.294, "p95_ms": 273.469}, "triton_wait": {"avg_ms": 67.898, "count": 122, "max_ms": 95.755, "p95_ms": 87.757}, "visualize": {"avg_ms": 11.02, "count": 122, "max_ms": 13.302, "p95_ms": 12.031}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.763, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 533}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4032.988, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:08:51+09:00", "last_error_summary": "[h264 @ 0x559c510dba00] cabac decode of qscale diff failed at 28 42 [h264 @ 0x559c510dba00] error while decoding MB 28 42, bytestream 2510 [h264 @ 0x559c5117ddc0] error while decoding MB 55 28, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 112, "h264_decode_error_count": 117, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:08:51+09:00", "last_error_summary": "[h264 @ 0x559c510dba00] cabac decode of qscale diff failed at 28 42 [h264 @ 0x559c510dba00] error while decoding MB 28 42, bytestream 2510 [h264 @ 0x559c5117ddc0] error while decoding MB 55 28, bytestream -9", "last_exit_at": "2026-07-01T19:08:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 104.447, "last_started_at": "2026-07-01T19:07:06+09:00", "pid": 17827, "process": {"exists": false, "pid": 17827}, "restart_count": 132, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 133}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.714, "sample_count": 1169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63964, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:08:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:08:51+09:00", "last_frame_read_age_sec": 0.257, "last_frame_read_at": "2026-07-01T19:08:51+09:00", "mjpeg": {"age_sec": 0.247, "exists": true, "mtime": "2026-07-01T19:08:51.134865+09:00", "mtime_epoch": 1782900531.1348646, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192382}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:08:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 891.332, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.105, "pushed": 4.105, "read": 4.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.15, "count": 122, "max_ms": 12.864, "p95_ms": 10.612}, "gstreamer_push": {"avg_ms": 0.934, "count": 123, "max_ms": 1.884, "p95_ms": 1.798}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.494, "count": 122, "max_ms": 65.929, "p95_ms": 56.589}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.249, "count": 122, "max_ms": 95.745, "p95_ms": 87.747}, "yolo11_640_fire_11class": {"avg_ms": 52.949, "count": 122, "max_ms": 73.454, "p95_ms": 69.379}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.062, "count": 122, "max_ms": 94.864, "p95_ms": 86.559}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.002, "count": 123, "max_ms": 298.172, "p95_ms": 263.434}, "read_interval": {"avg_ms": 245.524, "count": 122, "max_ms": 308.294, "p95_ms": 273.469}, "triton_wait": {"avg_ms": 67.779, "count": 122, "max_ms": 95.755, "p95_ms": 87.757}, "visualize": {"avg_ms": 11.021, "count": 122, "max_ms": 13.302, "p95_ms": 12.031}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 533}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4032.98, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 112, "h264_decode_error_count": 117, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:08:51+09:00", "last_error_summary": "[h264 @ 0x559c510dba00] cabac decode of qscale diff failed at 28 42 [h264 @ 0x559c510dba00] error while decoding MB 28 42, bytestream 2510 [h264 @ 0x559c5117ddc0] error while decoding MB 55 28, bytestream -9", "last_exit_at": "2026-07-01T19:08:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 104.45, "last_started_at": "2026-07-01T19:07:06+09:00", "pid": 17827, "process": {"exists": false, "pid": 17827}, "restart_count": 132, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 133}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.714, "sample_count": 1169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63964, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:08:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:08:51+09:00", "last_frame_read_age_sec": 0.26, "last_frame_read_at": "2026-07-01T19:08:51+09:00", "mjpeg": {"age_sec": 0.25, "exists": true, "mtime": "2026-07-01T19:08:51.134865+09:00", "mtime_epoch": 1782900531.1348646, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192382}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:08:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 891.332, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.104, "pushed": 4.104, "read": 4.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.15, "count": 122, "max_ms": 12.864, "p95_ms": 10.612}, "gstreamer_push": {"avg_ms": 0.934, "count": 123, "max_ms": 1.884, "p95_ms": 1.798}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.494, "count": 122, "max_ms": 65.929, "p95_ms": 56.589}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.249, "count": 122, "max_ms": 95.745, "p95_ms": 87.747}, "yolo11_640_fire_11class": {"avg_ms": 52.949, "count": 122, "max_ms": 73.454, "p95_ms": 69.379}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.062, "count": 122, "max_ms": 94.864, "p95_ms": 86.559}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.002, "count": 123, "max_ms": 298.172, "p95_ms": 263.434}, "read_interval": {"avg_ms": 245.524, "count": 122, "max_ms": 308.294, "p95_ms": 273.469}, "triton_wait": {"avg_ms": 67.779, "count": 122, "max_ms": 95.755, "p95_ms": 87.757}, "visualize": {"avg_ms": 11.021, "count": 122, "max_ms": 13.302, "p95_ms": 12.031}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 533}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4140.762, "limit_mb": 8796093022207.996}, "detail": {"pid": 17934, "restart_count": 133, "start_count": 134}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 112, "h264_decode_error_count": 117, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:08:51+09:00", "last_error_summary": "[h264 @ 0x559c510dba00] cabac decode of qscale diff failed at 28 42 [h264 @ 0x559c510dba00] error while decoding MB 28 42, bytestream 2510 [h264 @ 0x559c5117ddc0] error while decoding MB 55 28, bytestream -9", "last_exit_at": "2026-07-01T19:08:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:08:54+09:00", "pid": 17934, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.598, "name": "ffmpeg", "pid": 17934, "rss_mb": 138.598, "state": "S (sleeping)", "threads": 106}, "restart_count": 133, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 134}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.837, "sample_count": 1150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 63964, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:08:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.665, "last_frame_pushed_at": "2026-07-01T19:08:51+09:00", "last_frame_read_age_sec": 4.919, "last_frame_read_at": "2026-07-01T19:08:51+09:00", "mjpeg": {"age_sec": 4.649, "exists": true, "mtime": "2026-07-01T19:08:51.394868+09:00", "mtime_epoch": 1782900531.3948677, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192155}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:08:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 895.613, "name": "celery", "pid": 237, "rss_mb": 891.332, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.45, "pushed": 3.45, "read": 3.417}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.242, "count": 103, "max_ms": 12.864, "p95_ms": 10.713}, "gstreamer_push": {"avg_ms": 0.938, "count": 103, "max_ms": 1.884, "p95_ms": 1.798}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.525, "count": 103, "max_ms": 65.929, "p95_ms": 56.439}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.061, "count": 103, "max_ms": 95.745, "p95_ms": 85.791}, "yolo11_640_fire_11class": {"avg_ms": 52.881, "count": 103, "max_ms": 73.454, "p95_ms": 69.379}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.014, "count": 103, "max_ms": 94.864, "p95_ms": 85.952}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.089, "count": 103, "max_ms": 298.172, "p95_ms": 263.719}, "read_interval": {"avg_ms": 246.619, "count": 102, "max_ms": 308.294, "p95_ms": 276.618}, "triton_wait": {"avg_ms": 67.653, "count": 103, "max_ms": 95.755, "p95_ms": 86.906}, "visualize": {"avg_ms": 11.044, "count": 103, "max_ms": 13.302, "p95_ms": 12.031}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.854, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4032.496, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e85ed1f180] error while decoding MB 59 38, bytestream -24", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 113, "h264_decode_error_count": 118, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:10:42+09:00", "last_error_summary": "[h264 @ 0x55e85ed1f180] error while decoding MB 59 38, bytestream -24", "last_exit_at": "2026-07-01T19:08:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.688, "last_started_at": "2026-07-01T19:08:54+09:00", "pid": 17934, "process": {"exists": false, "pid": 17934}, "restart_count": 133, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 134}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.225, "sample_count": 1171, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64397, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:10:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T19:10:42+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T19:10:42+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T19:10:42.684170+09:00", "mtime_epoch": 1782900642.6841705, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191832}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:10:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 889.789, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.119, "pushed": 4.119, "read": 4.119}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.127, "count": 122, "max_ms": 13.666, "p95_ms": 10.241}, "gstreamer_push": {"avg_ms": 0.875, "count": 123, "max_ms": 1.899, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.913, "count": 122, "max_ms": 68.773, "p95_ms": 61.448}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.559, "count": 122, "max_ms": 96.559, "p95_ms": 88.024}, "yolo11_640_fire_11class": {"avg_ms": 54.724, "count": 122, "max_ms": 76.987, "p95_ms": 71.394}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.545, "count": 122, "max_ms": 95.391, "p95_ms": 89.476}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.802, "count": 123, "max_ms": 298.518, "p95_ms": 270.094}, "read_interval": {"avg_ms": 244.135, "count": 123, "max_ms": 307.418, "p95_ms": 279.509}, "triton_wait": {"avg_ms": 69.418, "count": 122, "max_ms": 96.571, "p95_ms": 89.879}, "visualize": {"avg_ms": 10.901, "count": 122, "max_ms": 12.325, "p95_ms": 11.683}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.861, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4034.293, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:10:42+09:00", "last_error_summary": "[h264 @ 0x55e85ed1f180] error while decoding MB 59 38, bytestream -24", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 113, "h264_decode_error_count": 118, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:10:42+09:00", "last_error_summary": "[h264 @ 0x55e85ed1f180] error while decoding MB 59 38, bytestream -24", "last_exit_at": "2026-07-01T19:10:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.869, "last_started_at": "2026-07-01T19:08:54+09:00", "pid": 17934, "process": {"exists": false, "pid": 17934}, "restart_count": 133, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 134}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.964, "sample_count": 1172, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64398, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:10:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:10:42+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T19:10:42+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T19:10:42.684170+09:00", "mtime_epoch": 1782900642.6841705, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191832}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:10:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 892.367, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.129, "pushed": 4.129, "read": 4.095}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.149, "count": 123, "max_ms": 13.666, "p95_ms": 10.265}, "gstreamer_push": {"avg_ms": 0.875, "count": 123, "max_ms": 1.899, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.828, "count": 123, "max_ms": 68.773, "p95_ms": 61.448}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.49, "count": 123, "max_ms": 96.559, "p95_ms": 88.024}, "yolo11_640_fire_11class": {"avg_ms": 54.65, "count": 123, "max_ms": 76.987, "p95_ms": 71.394}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.457, "count": 123, "max_ms": 95.391, "p95_ms": 89.476}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.874, "count": 123, "max_ms": 298.518, "p95_ms": 270.094}, "read_interval": {"avg_ms": 244.247, "count": 122, "max_ms": 307.418, "p95_ms": 279.509}, "triton_wait": {"avg_ms": 69.343, "count": 123, "max_ms": 96.571, "p95_ms": 89.879}, "visualize": {"avg_ms": 10.899, "count": 123, "max_ms": 12.325, "p95_ms": 11.683}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.79, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4034.285, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 113, "h264_decode_error_count": 118, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:10:42+09:00", "last_error_summary": "[h264 @ 0x55e85ed1f180] error while decoding MB 59 38, bytestream -24", "last_exit_at": "2026-07-01T19:10:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.872, "last_started_at": "2026-07-01T19:08:54+09:00", "pid": 17934, "process": {"exists": false, "pid": 17934}, "restart_count": 133, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 134}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 273.964, "sample_count": 1172, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64398, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:10:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:10:42+09:00", "last_frame_read_age_sec": 0.237, "last_frame_read_at": "2026-07-01T19:10:42+09:00", "mjpeg": {"age_sec": 0.231, "exists": true, "mtime": "2026-07-01T19:10:42.684170+09:00", "mtime_epoch": 1782900642.6841705, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191832}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:10:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 892.367, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.128, "pushed": 4.128, "read": 4.095}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.149, "count": 123, "max_ms": 13.666, "p95_ms": 10.265}, "gstreamer_push": {"avg_ms": 0.875, "count": 123, "max_ms": 1.899, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.828, "count": 123, "max_ms": 68.773, "p95_ms": 61.448}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.49, "count": 123, "max_ms": 96.559, "p95_ms": 88.024}, "yolo11_640_fire_11class": {"avg_ms": 54.65, "count": 123, "max_ms": 76.987, "p95_ms": 71.394}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.457, "count": 123, "max_ms": 95.391, "p95_ms": 89.476}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.874, "count": 123, "max_ms": 298.518, "p95_ms": 270.094}, "read_interval": {"avg_ms": 244.247, "count": 122, "max_ms": 307.418, "p95_ms": 279.509}, "triton_wait": {"avg_ms": 69.343, "count": 123, "max_ms": 96.571, "p95_ms": 89.879}, "visualize": {"avg_ms": 10.899, "count": 123, "max_ms": 12.325, "p95_ms": 11.683}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.793, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4140.363, "limit_mb": 8796093022207.996}, "detail": {"pid": 18041, "restart_count": 134, "start_count": 135}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 113, "h264_decode_error_count": 118, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:10:42+09:00", "last_error_summary": "[h264 @ 0x55e85ed1f180] error while decoding MB 59 38, bytestream -24", "last_exit_at": "2026-07-01T19:10:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:10:45+09:00", "pid": 18041, "process": {"exists": true, "fd_count": 5, "hwm_mb": 135.953, "name": "ffmpeg", "pid": 18041, "rss_mb": 135.953, "state": "S (sleeping)", "threads": 106}, "restart_count": 134, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 135}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.225, "sample_count": 1152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64398, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:10:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.823, "last_frame_pushed_at": "2026-07-01T19:10:42+09:00", "last_frame_read_age_sec": 5.054, "last_frame_read_at": "2026-07-01T19:10:42+09:00", "mjpeg": {"age_sec": 4.808, "exists": true, "mtime": "2026-07-01T19:10:42.924173+09:00", "mtime_epoch": 1782900642.9241734, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191766}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:10:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 892.367, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.502, "pushed": 3.502, "read": 3.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.089, "count": 104, "max_ms": 13.666, "p95_ms": 10.241}, "gstreamer_push": {"avg_ms": 0.877, "count": 105, "max_ms": 1.899, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.323, "count": 104, "max_ms": 68.773, "p95_ms": 61.539}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.302, "count": 104, "max_ms": 91.362, "p95_ms": 86.124}, "yolo11_640_fire_11class": {"avg_ms": 53.611, "count": 104, "max_ms": 72.66, "p95_ms": 71.394}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.268, "count": 104, "max_ms": 90.199, "p95_ms": 88.487}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.909, "count": 105, "max_ms": 298.518, "p95_ms": 268.337}, "read_interval": {"avg_ms": 242.252, "count": 104, "max_ms": 307.418, "p95_ms": 278.368}, "triton_wait": {"avg_ms": 68.109, "count": 104, "max_ms": 91.373, "p95_ms": 88.791}, "visualize": {"avg_ms": 10.869, "count": 104, "max_ms": 12.325, "p95_ms": 11.461}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 540}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4033.281, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56351f976c00] error while decoding MB 68 17, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 114, "h264_decode_error_count": 119, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:12:35+09:00", "last_error_summary": "[h264 @ 0x56351f976c00] error while decoding MB 68 17, bytestream -7", "last_exit_at": "2026-07-01T19:10:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.94, "last_started_at": "2026-07-01T19:10:45+09:00", "pid": 18041, "process": {"exists": false, "pid": 18041}, "restart_count": 134, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 135}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.308, "sample_count": 1180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64837, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:12:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T19:12:35+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T19:12:35+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T19:12:35.633568+09:00", "mtime_epoch": 1782900755.6335683, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189239}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:12:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 889.707, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.031, "pushed": 4.031, "read": 4.031}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.334, "count": 120, "max_ms": 10.768, "p95_ms": 9.917}, "gstreamer_push": {"avg_ms": 0.978, "count": 120, "max_ms": 2.886, "p95_ms": 1.837}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.83, "count": 120, "max_ms": 72.213, "p95_ms": 62.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.836, "count": 120, "max_ms": 92.987, "p95_ms": 88.097}, "yolo11_640_fire_11class": {"avg_ms": 54.354, "count": 120, "max_ms": 75.199, "p95_ms": 71.78}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.512, "count": 120, "max_ms": 93.331, "p95_ms": 90.093}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.308, "count": 120, "max_ms": 344.415, "p95_ms": 277.509}, "read_interval": {"avg_ms": 249.911, "count": 120, "max_ms": 353.506, "p95_ms": 286.632}, "triton_wait": {"avg_ms": 69.604, "count": 120, "max_ms": 93.344, "p95_ms": 90.536}, "visualize": {"avg_ms": 11.127, "count": 120, "max_ms": 12.87, "p95_ms": 12.09}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.771, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 540}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4032.383, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:12:35+09:00", "last_error_summary": "[h264 @ 0x56351f976c00] error while decoding MB 68 17, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 114, "h264_decode_error_count": 119, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:12:35+09:00", "last_error_summary": "[h264 @ 0x56351f976c00] error while decoding MB 68 17, bytestream -7", "last_exit_at": "2026-07-01T19:12:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.123, "last_started_at": "2026-07-01T19:10:45+09:00", "pid": 18041, "process": {"exists": false, "pid": 18041}, "restart_count": 134, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 135}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.308, "sample_count": 1180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64838, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:12:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:12:35+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T19:12:35+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-01T19:12:35.633568+09:00", "mtime_epoch": 1782900755.6335683, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189239}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:12:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 889.707, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.039, "pushed": 4.039, "read": 4.006}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.327, "count": 120, "max_ms": 10.768, "p95_ms": 9.917}, "gstreamer_push": {"avg_ms": 0.977, "count": 121, "max_ms": 2.886, "p95_ms": 1.837}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.799, "count": 120, "max_ms": 72.213, "p95_ms": 62.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.707, "count": 120, "max_ms": 92.987, "p95_ms": 88.097}, "yolo11_640_fire_11class": {"avg_ms": 54.25, "count": 120, "max_ms": 75.199, "p95_ms": 71.78}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.399, "count": 120, "max_ms": 93.331, "p95_ms": 90.093}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.221, "count": 121, "max_ms": 344.415, "p95_ms": 277.509}, "read_interval": {"avg_ms": 249.911, "count": 120, "max_ms": 353.506, "p95_ms": 286.632}, "triton_wait": {"avg_ms": 69.484, "count": 120, "max_ms": 93.344, "p95_ms": 90.536}, "visualize": {"avg_ms": 11.114, "count": 120, "max_ms": 12.87, "p95_ms": 12.09}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 540}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4032.375, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 114, "h264_decode_error_count": 119, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:12:35+09:00", "last_error_summary": "[h264 @ 0x56351f976c00] error while decoding MB 68 17, bytestream -7", "last_exit_at": "2026-07-01T19:12:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.126, "last_started_at": "2026-07-01T19:10:45+09:00", "pid": 18041, "process": {"exists": false, "pid": 18041}, "restart_count": 134, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 135}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.308, "sample_count": 1180, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64838, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:12:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:12:35+09:00", "last_frame_read_age_sec": 0.237, "last_frame_read_at": "2026-07-01T19:12:35+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T19:12:35.633568+09:00", "mtime_epoch": 1782900755.6335683, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189239}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:12:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 889.707, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.039, "pushed": 4.039, "read": 4.006}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.327, "count": 120, "max_ms": 10.768, "p95_ms": 9.917}, "gstreamer_push": {"avg_ms": 0.977, "count": 121, "max_ms": 2.886, "p95_ms": 1.837}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.799, "count": 120, "max_ms": 72.213, "p95_ms": 62.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.707, "count": 120, "max_ms": 92.987, "p95_ms": 88.097}, "yolo11_640_fire_11class": {"avg_ms": 54.25, "count": 120, "max_ms": 75.199, "p95_ms": 71.78}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.399, "count": 120, "max_ms": 93.331, "p95_ms": 90.093}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.221, "count": 121, "max_ms": 344.415, "p95_ms": 277.509}, "read_interval": {"avg_ms": 249.911, "count": 120, "max_ms": 353.506, "p95_ms": 286.632}, "triton_wait": {"avg_ms": 69.484, "count": 120, "max_ms": 93.344, "p95_ms": 90.536}, "visualize": {"avg_ms": 11.114, "count": 120, "max_ms": 12.87, "p95_ms": 12.09}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 540}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4140.16, "limit_mb": 8796093022207.996}, "detail": {"pid": 18148, "restart_count": 135, "start_count": 136}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 114, "h264_decode_error_count": 119, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:12:35+09:00", "last_error_summary": "[h264 @ 0x56351f976c00] error while decoding MB 68 17, bytestream -7", "last_exit_at": "2026-07-01T19:12:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:12:37+09:00", "pid": 18148, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.105, "name": "ffmpeg", "pid": 18148, "rss_mb": 139.105, "state": "S (sleeping)", "threads": 106}, "restart_count": 135, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 136}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.343, "sample_count": 1164, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64838, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:12:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.077, "last_frame_pushed_at": "2026-07-01T19:12:35+09:00", "last_frame_read_age_sec": 4.308, "last_frame_read_at": "2026-07-01T19:12:35+09:00", "mjpeg": {"age_sec": 4.054, "exists": true, "mtime": "2026-07-01T19:12:35.873571+09:00", "mtime_epoch": 1782900755.8735714, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186659}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:12:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 889.707, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.504, "pushed": 3.504, "read": 3.471}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.271, "count": 104, "max_ms": 10.302, "p95_ms": 9.739}, "gstreamer_push": {"avg_ms": 0.956, "count": 105, "max_ms": 2.886, "p95_ms": 1.828}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.91, "count": 104, "max_ms": 72.213, "p95_ms": 61.978}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.842, "count": 104, "max_ms": 92.987, "p95_ms": 88.132}, "yolo11_640_fire_11class": {"avg_ms": 54.446, "count": 104, "max_ms": 75.199, "p95_ms": 71.928}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.565, "count": 104, "max_ms": 93.331, "p95_ms": 89.914}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.152, "count": 105, "max_ms": 344.415, "p95_ms": 277.16}, "read_interval": {"avg_ms": 248.817, "count": 104, "max_ms": 353.506, "p95_ms": 286.325}, "triton_wait": {"avg_ms": 69.609, "count": 104, "max_ms": 93.344, "p95_ms": 90.105}, "visualize": {"avg_ms": 11.073, "count": 104, "max_ms": 12.743, "p95_ms": 12.063}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 543}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4036.801, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e401c20100] error while decoding MB 5 19, bytestream -40", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 115, "h264_decode_error_count": 120, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:14:28+09:00", "last_error_summary": "[h264 @ 0x55e401c20100] error while decoding MB 5 19, bytestream -40", "last_exit_at": "2026-07-01T19:12:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.128, "last_started_at": "2026-07-01T19:12:37+09:00", "pid": 18148, "process": {"exists": false, "pid": 18148}, "restart_count": 135, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 136}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.455, "sample_count": 1182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 65272, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:14:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T19:14:27+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T19:14:28+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T19:14:28.015020+09:00", "mtime_epoch": 1782900868.01502, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189537}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:14:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 889.488, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.009, "pushed": 4.009, "read": 4.009}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.516, "count": 119, "max_ms": 15.572, "p95_ms": 12.371}, "gstreamer_push": {"avg_ms": 0.912, "count": 120, "max_ms": 2.734, "p95_ms": 1.011}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.525, "count": 119, "max_ms": 76.251, "p95_ms": 65.39}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.33, "count": 119, "max_ms": 94.115, "p95_ms": 89.827}, "yolo11_640_fire_11class": {"avg_ms": 53.977, "count": 119, "max_ms": 76.791, "p95_ms": 71.174}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.628, "count": 119, "max_ms": 93.029, "p95_ms": 90.056}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.456, "count": 120, "max_ms": 344.338, "p95_ms": 282.265}, "read_interval": {"avg_ms": 251.921, "count": 120, "max_ms": 356.375, "p95_ms": 293.607}, "triton_wait": {"avg_ms": 69.901, "count": 119, "max_ms": 94.126, "p95_ms": 90.531}, "visualize": {"avg_ms": 11.531, "count": 119, "max_ms": 16.998, "p95_ms": 15.422}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 543}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4035.91, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:14:28+09:00", "last_error_summary": "[h264 @ 0x55e401c20100] error while decoding MB 5 19, bytestream -40", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 115, "h264_decode_error_count": 120, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:14:28+09:00", "last_error_summary": "[h264 @ 0x55e401c20100] error while decoding MB 5 19, bytestream -40", "last_exit_at": "2026-07-01T19:14:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.338, "last_started_at": "2026-07-01T19:12:37+09:00", "pid": 18148, "process": {"exists": false, "pid": 18148}, "restart_count": 135, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 136}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.455, "sample_count": 1182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 65273, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:14:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T19:14:28+09:00", "last_frame_read_age_sec": 0.26, "last_frame_read_at": "2026-07-01T19:14:28+09:00", "mjpeg": {"age_sec": 0.25, "exists": true, "mtime": "2026-07-01T19:14:28.015020+09:00", "mtime_epoch": 1782900868.01502, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189537}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:14:28+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 889.488, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.016, "pushed": 4.016, "read": 3.983}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.507, "count": 119, "max_ms": 15.572, "p95_ms": 12.371}, "gstreamer_push": {"avg_ms": 0.904, "count": 120, "max_ms": 2.734, "p95_ms": 0.946}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.624, "count": 119, "max_ms": 76.251, "p95_ms": 65.39}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.536, "count": 119, "max_ms": 94.115, "p95_ms": 90.079}, "yolo11_640_fire_11class": {"avg_ms": 54.157, "count": 119, "max_ms": 76.791, "p95_ms": 71.946}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.817, "count": 119, "max_ms": 93.029, "p95_ms": 90.127}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.704, "count": 120, "max_ms": 321.406, "p95_ms": 275.304}, "read_interval": {"avg_ms": 251.043, "count": 119, "max_ms": 332.71, "p95_ms": 293.607}, "triton_wait": {"avg_ms": 70.097, "count": 119, "max_ms": 94.126, "p95_ms": 90.824}, "visualize": {"avg_ms": 11.518, "count": 119, "max_ms": 16.998, "p95_ms": 15.422}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 543}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4035.84, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 115, "h264_decode_error_count": 120, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:14:28+09:00", "last_error_summary": "[h264 @ 0x55e401c20100] error while decoding MB 5 19, bytestream -40", "last_exit_at": "2026-07-01T19:14:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.345, "last_started_at": "2026-07-01T19:12:37+09:00", "pid": 18148, "process": {"exists": false, "pid": 18148}, "restart_count": 135, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 136}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.455, "sample_count": 1182, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 65273, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:14:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T19:14:28+09:00", "last_frame_read_age_sec": 0.267, "last_frame_read_at": "2026-07-01T19:14:28+09:00", "mjpeg": {"age_sec": 0.257, "exists": true, "mtime": "2026-07-01T19:14:28.015020+09:00", "mtime_epoch": 1782900868.01502, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189537}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:14:28+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 889.488, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.015, "pushed": 4.015, "read": 3.982}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.507, "count": 119, "max_ms": 15.572, "p95_ms": 12.371}, "gstreamer_push": {"avg_ms": 0.904, "count": 120, "max_ms": 2.734, "p95_ms": 0.946}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.624, "count": 119, "max_ms": 76.251, "p95_ms": 65.39}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.536, "count": 119, "max_ms": 94.115, "p95_ms": 90.079}, "yolo11_640_fire_11class": {"avg_ms": 54.157, "count": 119, "max_ms": 76.791, "p95_ms": 71.946}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.817, "count": 119, "max_ms": 93.029, "p95_ms": 90.127}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.704, "count": 120, "max_ms": 321.406, "p95_ms": 275.304}, "read_interval": {"avg_ms": 251.043, "count": 119, "max_ms": 332.71, "p95_ms": 293.607}, "triton_wait": {"avg_ms": 70.097, "count": 119, "max_ms": 94.126, "p95_ms": 90.824}, "visualize": {"avg_ms": 11.518, "count": 119, "max_ms": 16.998, "p95_ms": 15.422}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 543}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4141.855, "limit_mb": 8796093022207.996}, "detail": {"pid": 18255, "restart_count": 136, "start_count": 137}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 115, "h264_decode_error_count": 120, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:14:28+09:00", "last_error_summary": "[h264 @ 0x55e401c20100] error while decoding MB 5 19, bytestream -40", "last_exit_at": "2026-07-01T19:14:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:14:31+09:00", "pid": 18255, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.867, "name": "ffmpeg", "pid": 18255, "rss_mb": 138.867, "state": "S (sleeping)", "threads": 106}, "restart_count": 136, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 137}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.466, "sample_count": 1162, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 65273, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:14:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.812, "last_frame_pushed_at": "2026-07-01T19:14:28+09:00", "last_frame_read_age_sec": 5.068, "last_frame_read_at": "2026-07-01T19:14:28+09:00", "mjpeg": {"age_sec": 4.794, "exists": true, "mtime": "2026-07-01T19:14:28.279023+09:00", "mtime_epoch": 1782900868.2790234, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:14:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 889.488, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.39, "pushed": 3.39, "read": 3.357}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.508, "count": 101, "max_ms": 15.572, "p95_ms": 12.371}, "gstreamer_push": {"avg_ms": 0.91, "count": 101, "max_ms": 2.734, "p95_ms": 0.946}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.387, "count": 101, "max_ms": 76.251, "p95_ms": 65.39}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.099, "count": 101, "max_ms": 94.115, "p95_ms": 90.079}, "yolo11_640_fire_11class": {"avg_ms": 54.554, "count": 101, "max_ms": 76.791, "p95_ms": 71.946}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.384, "count": 101, "max_ms": 93.029, "p95_ms": 90.127}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.43, "count": 101, "max_ms": 321.406, "p95_ms": 282.265}, "read_interval": {"avg_ms": 249.665, "count": 100, "max_ms": 332.71, "p95_ms": 293.607}, "triton_wait": {"avg_ms": 70.682, "count": 101, "max_ms": 94.126, "p95_ms": 90.824}, "visualize": {"avg_ms": 11.419, "count": 101, "max_ms": 15.933, "p95_ms": 15.186}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.79, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 548}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4041.309, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56233d3c2e40] left block unavailable for requested intra mode [h264 @ 0x56233d3c2e40] error while decoding MB 0 5, bytestream 107670", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 116, "h264_decode_error_count": 121, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:16:49+09:00", "last_error_summary": "[h264 @ 0x56233d3c2e40] left block unavailable for requested intra mode [h264 @ 0x56233d3c2e40] error while decoding MB 0 5, bytestream 107670", "last_exit_at": "2026-07-01T19:14:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.213, "last_started_at": "2026-07-01T19:14:31+09:00", "pid": 18255, "process": {"exists": false, "pid": 18255}, "restart_count": 136, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 137}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.736, "sample_count": 1157, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 65808, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:16:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T19:16:49+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T19:16:49+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T19:16:49.240911+09:00", "mtime_epoch": 1782901009.2409115, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199384}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:16:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 893.566, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.904, "pushed": 3.904, "read": 3.904}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.654, "count": 116, "max_ms": 15.602, "p95_ms": 10.575}, "gstreamer_push": {"avg_ms": 0.977, "count": 117, "max_ms": 2.889, "p95_ms": 1.865}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.276, "count": 116, "max_ms": 71.95, "p95_ms": 69.165}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.92, "count": 116, "max_ms": 96.51, "p95_ms": 91.39}, "yolo11_640_fire_11class": {"avg_ms": 56.304, "count": 116, "max_ms": 77.964, "p95_ms": 73.199}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.353, "count": 116, "max_ms": 95.596, "p95_ms": 90.455}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.031, "count": 117, "max_ms": 349.489, "p95_ms": 279.782}, "read_interval": {"avg_ms": 257.738, "count": 117, "max_ms": 360.641, "p95_ms": 290.138}, "triton_wait": {"avg_ms": 72.774, "count": 116, "max_ms": 96.523, "p95_ms": 91.777}, "visualize": {"avg_ms": 11.396, "count": 116, "max_ms": 15.077, "p95_ms": 12.865}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 548}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4040.332, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:16:49+09:00", "last_error_summary": "[h264 @ 0x56233d3c2e40] left block unavailable for requested intra mode [h264 @ 0x56233d3c2e40] error while decoding MB 0 5, bytestream 107670", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 116, "h264_decode_error_count": 121, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:16:49+09:00", "last_error_summary": "[h264 @ 0x56233d3c2e40] left block unavailable for requested intra mode [h264 @ 0x56233d3c2e40] error while decoding MB 0 5, bytestream 107670", "last_exit_at": "2026-07-01T19:16:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.388, "last_started_at": "2026-07-01T19:14:31+09:00", "pid": 18255, "process": {"exists": false, "pid": 18255}, "restart_count": 136, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 137}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.736, "sample_count": 1157, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 65809, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:16:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:16:49+09:00", "last_frame_read_age_sec": 0.227, "last_frame_read_at": "2026-07-01T19:16:49+09:00", "mjpeg": {"age_sec": 0.218, "exists": true, "mtime": "2026-07-01T19:16:49.240911+09:00", "mtime_epoch": 1782901009.2409115, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199384}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:16:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 893.566, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.914, "pushed": 3.914, "read": 3.88}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.637, "count": 116, "max_ms": 15.602, "p95_ms": 10.567}, "gstreamer_push": {"avg_ms": 0.977, "count": 117, "max_ms": 2.889, "p95_ms": 1.865}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.373, "count": 116, "max_ms": 71.95, "p95_ms": 69.165}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.981, "count": 116, "max_ms": 96.51, "p95_ms": 91.39}, "yolo11_640_fire_11class": {"avg_ms": 56.398, "count": 116, "max_ms": 77.964, "p95_ms": 73.199}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.415, "count": 116, "max_ms": 95.596, "p95_ms": 90.455}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.918, "count": 117, "max_ms": 349.489, "p95_ms": 279.782}, "read_interval": {"avg_ms": 257.849, "count": 116, "max_ms": 360.641, "p95_ms": 290.138}, "triton_wait": {"avg_ms": 72.836, "count": 116, "max_ms": 96.523, "p95_ms": 91.777}, "visualize": {"avg_ms": 11.366, "count": 116, "max_ms": 15.077, "p95_ms": 12.747}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 548}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4040.324, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 116, "h264_decode_error_count": 121, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:16:49+09:00", "last_error_summary": "[h264 @ 0x56233d3c2e40] left block unavailable for requested intra mode [h264 @ 0x56233d3c2e40] error while decoding MB 0 5, bytestream 107670", "last_exit_at": "2026-07-01T19:16:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.391, "last_started_at": "2026-07-01T19:14:31+09:00", "pid": 18255, "process": {"exists": false, "pid": 18255}, "restart_count": 136, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 137}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.736, "sample_count": 1157, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 65809, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:16:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:16:49+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T19:16:49+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T19:16:49.240911+09:00", "mtime_epoch": 1782901009.2409115, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199384}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:16:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 893.566, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.913, "pushed": 3.913, "read": 3.88}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.637, "count": 116, "max_ms": 15.602, "p95_ms": 10.567}, "gstreamer_push": {"avg_ms": 0.977, "count": 117, "max_ms": 2.889, "p95_ms": 1.865}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.373, "count": 116, "max_ms": 71.95, "p95_ms": 69.165}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.981, "count": 116, "max_ms": 96.51, "p95_ms": 91.39}, "yolo11_640_fire_11class": {"avg_ms": 56.398, "count": 116, "max_ms": 77.964, "p95_ms": 73.199}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.415, "count": 116, "max_ms": 95.596, "p95_ms": 90.455}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.918, "count": 117, "max_ms": 349.489, "p95_ms": 279.782}, "read_interval": {"avg_ms": 257.849, "count": 116, "max_ms": 360.641, "p95_ms": 290.138}, "triton_wait": {"avg_ms": 72.836, "count": 116, "max_ms": 96.523, "p95_ms": 91.777}, "visualize": {"avg_ms": 11.366, "count": 116, "max_ms": 15.077, "p95_ms": 12.747}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 548}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4153.453, "limit_mb": 8796093022207.996}, "detail": {"pid": 18362, "restart_count": 137, "start_count": 138}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 116, "h264_decode_error_count": 121, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:16:49+09:00", "last_error_summary": "[h264 @ 0x56233d3c2e40] left block unavailable for requested intra mode [h264 @ 0x56233d3c2e40] error while decoding MB 0 5, bytestream 107670", "last_exit_at": "2026-07-01T19:16:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:16:51+09:00", "pid": 18362, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.141, "name": "ffmpeg", "pid": 18362, "rss_mb": 142.141, "state": "S (sleeping)", "threads": 106}, "restart_count": 137, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 138}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.736, "sample_count": 1140, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 65809, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:16:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.129, "last_frame_pushed_at": "2026-07-01T19:16:49+09:00", "last_frame_read_age_sec": 4.353, "last_frame_read_at": "2026-07-01T19:16:49+09:00", "mjpeg": {"age_sec": 4.113, "exists": true, "mtime": "2026-07-01T19:16:49.472915+09:00", "mtime_epoch": 1782901009.4729147, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199795}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:16:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 893.566, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.413, "pushed": 3.413, "read": 3.379}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.549, "count": 101, "max_ms": 13.93, "p95_ms": 10.502}, "gstreamer_push": {"avg_ms": 0.971, "count": 102, "max_ms": 2.889, "p95_ms": 1.798}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.413, "count": 101, "max_ms": 71.95, "p95_ms": 68.575}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.446, "count": 101, "max_ms": 96.51, "p95_ms": 91.39}, "yolo11_640_fire_11class": {"avg_ms": 55.956, "count": 101, "max_ms": 77.964, "p95_ms": 73.199}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.994, "count": 101, "max_ms": 95.596, "p95_ms": 90.113}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.416, "count": 102, "max_ms": 349.489, "p95_ms": 278.045}, "read_interval": {"avg_ms": 255.314, "count": 101, "max_ms": 360.641, "p95_ms": 288.587}, "triton_wait": {"avg_ms": 72.347, "count": 101, "max_ms": 96.523, "p95_ms": 91.777}, "visualize": {"avg_ms": 11.304, "count": 101, "max_ms": 15.077, "p95_ms": 12.525}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4039.953, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5573d1cdd8c0] error while decoding MB 45 25, bytestream -28", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 117, "h264_decode_error_count": 122, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:18:42+09:00", "last_error_summary": "[h264 @ 0x5573d1cdd8c0] error while decoding MB 45 25, bytestream -28", "last_exit_at": "2026-07-01T19:16:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.859, "last_started_at": "2026-07-01T19:16:51+09:00", "pid": 18362, "process": {"exists": false, "pid": 18362}, "restart_count": 137, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 138}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.997, "sample_count": 1146, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 66237, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:18:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.054, "last_frame_pushed_at": "2026-07-01T19:18:42+09:00", "last_frame_read_age_sec": 0.045, "last_frame_read_at": "2026-07-01T19:18:42+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-01T19:18:42.406470+09:00", "mtime_epoch": 1782901122.4064703, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186805}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:18:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 890.918, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.917, "pushed": 3.917, "read": 3.917}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.853, "count": 116, "max_ms": 11.846, "p95_ms": 10.838}, "gstreamer_push": {"avg_ms": 0.915, "count": 117, "max_ms": 2.0, "p95_ms": 0.991}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.211, "count": 116, "max_ms": 71.356, "p95_ms": 62.348}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.124, "count": 116, "max_ms": 96.296, "p95_ms": 87.884}, "yolo11_640_fire_11class": {"avg_ms": 53.182, "count": 116, "max_ms": 74.035, "p95_ms": 72.122}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.895, "count": 116, "max_ms": 96.194, "p95_ms": 89.394}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.355, "count": 117, "max_ms": 344.391, "p95_ms": 274.997}, "read_interval": {"avg_ms": 256.936, "count": 117, "max_ms": 354.485, "p95_ms": 285.298}, "triton_wait": {"avg_ms": 68.893, "count": 116, "max_ms": 96.309, "p95_ms": 90.095}, "visualize": {"avg_ms": 11.672, "count": 116, "max_ms": 15.318, "p95_ms": 12.836}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4038.68, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:18:42+09:00", "last_error_summary": "[h264 @ 0x5573d1cdd8c0] error while decoding MB 45 25, bytestream -28", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 117, "h264_decode_error_count": 122, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:18:42+09:00", "last_error_summary": "[h264 @ 0x5573d1cdd8c0] error while decoding MB 45 25, bytestream -28", "last_exit_at": "2026-07-01T19:18:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.054, "last_started_at": "2026-07-01T19:16:51+09:00", "pid": 18362, "process": {"exists": false, "pid": 18362}, "restart_count": 137, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 138}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.997, "sample_count": 1146, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 66238, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:18:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:18:42+09:00", "last_frame_read_age_sec": 0.24, "last_frame_read_at": "2026-07-01T19:18:42+09:00", "mjpeg": {"age_sec": 0.232, "exists": true, "mtime": "2026-07-01T19:18:42.406470+09:00", "mtime_epoch": 1782901122.4064703, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186805}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:18:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 890.918, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.924, "pushed": 3.924, "read": 3.891}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.851, "count": 117, "max_ms": 11.846, "p95_ms": 10.838}, "gstreamer_push": {"avg_ms": 0.915, "count": 117, "max_ms": 2.0, "p95_ms": 0.991}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.232, "count": 117, "max_ms": 71.356, "p95_ms": 62.348}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.123, "count": 117, "max_ms": 96.296, "p95_ms": 87.884}, "yolo11_640_fire_11class": {"avg_ms": 53.196, "count": 117, "max_ms": 74.035, "p95_ms": 72.122}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.903, "count": 117, "max_ms": 96.194, "p95_ms": 89.394}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.314, "count": 117, "max_ms": 344.391, "p95_ms": 274.997}, "read_interval": {"avg_ms": 256.994, "count": 116, "max_ms": 354.485, "p95_ms": 285.298}, "triton_wait": {"avg_ms": 68.892, "count": 117, "max_ms": 96.309, "p95_ms": 90.095}, "visualize": {"avg_ms": 11.695, "count": 117, "max_ms": 15.318, "p95_ms": 12.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.814, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4038.672, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 117, "h264_decode_error_count": 122, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:18:42+09:00", "last_error_summary": "[h264 @ 0x5573d1cdd8c0] error while decoding MB 45 25, bytestream -28", "last_exit_at": "2026-07-01T19:18:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.057, "last_started_at": "2026-07-01T19:16:51+09:00", "pid": 18362, "process": {"exists": false, "pid": 18362}, "restart_count": 137, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 138}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.997, "sample_count": 1146, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 66238, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:18:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:18:42+09:00", "last_frame_read_age_sec": 0.244, "last_frame_read_at": "2026-07-01T19:18:42+09:00", "mjpeg": {"age_sec": 0.235, "exists": true, "mtime": "2026-07-01T19:18:42.406470+09:00", "mtime_epoch": 1782901122.4064703, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186805}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:18:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 890.918, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.924, "pushed": 3.924, "read": 3.89}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.851, "count": 117, "max_ms": 11.846, "p95_ms": 10.838}, "gstreamer_push": {"avg_ms": 0.915, "count": 117, "max_ms": 2.0, "p95_ms": 0.991}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.232, "count": 117, "max_ms": 71.356, "p95_ms": 62.348}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.123, "count": 117, "max_ms": 96.296, "p95_ms": 87.884}, "yolo11_640_fire_11class": {"avg_ms": 53.196, "count": 117, "max_ms": 74.035, "p95_ms": 72.122}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.903, "count": 117, "max_ms": 96.194, "p95_ms": 89.394}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.314, "count": 117, "max_ms": 344.391, "p95_ms": 274.997}, "read_interval": {"avg_ms": 256.994, "count": 116, "max_ms": 354.485, "p95_ms": 285.298}, "triton_wait": {"avg_ms": 68.892, "count": 117, "max_ms": 96.309, "p95_ms": 90.095}, "visualize": {"avg_ms": 11.695, "count": 117, "max_ms": 15.318, "p95_ms": 12.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.817, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4146.629, "limit_mb": 8796093022207.996}, "detail": {"pid": 18469, "restart_count": 138, "start_count": 139}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 117, "h264_decode_error_count": 122, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:18:42+09:00", "last_error_summary": "[h264 @ 0x5573d1cdd8c0] error while decoding MB 45 25, bytestream -28", "last_exit_at": "2026-07-01T19:18:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:18:45+09:00", "pid": 18469, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.199, "name": "ffmpeg", "pid": 18469, "rss_mb": 139.199, "state": "S (sleeping)", "threads": 106}, "restart_count": 138, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 139}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.839, "sample_count": 1126, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 66238, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:18:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 5.001, "last_frame_pushed_at": "2026-07-01T19:18:42+09:00", "last_frame_read_age_sec": 5.238, "last_frame_read_at": "2026-07-01T19:18:42+09:00", "mjpeg": {"age_sec": 4.982, "exists": true, "mtime": "2026-07-01T19:18:42.654474+09:00", "mtime_epoch": 1782901122.6544738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186756}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:18:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 890.918, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.309, "pushed": 3.309, "read": 3.275}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.909, "count": 98, "max_ms": 11.353, "p95_ms": 10.838}, "gstreamer_push": {"avg_ms": 0.923, "count": 99, "max_ms": 2.0, "p95_ms": 1.831}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.84, "count": 98, "max_ms": 67.671, "p95_ms": 58.877}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.548, "count": 98, "max_ms": 92.19, "p95_ms": 86.437}, "yolo11_640_fire_11class": {"avg_ms": 51.689, "count": 98, "max_ms": 73.261, "p95_ms": 69.04}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.183, "count": 98, "max_ms": 90.888, "p95_ms": 86.379}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.905, "count": 99, "max_ms": 344.391, "p95_ms": 272.667}, "read_interval": {"avg_ms": 254.497, "count": 98, "max_ms": 354.485, "p95_ms": 283.173}, "triton_wait": {"avg_ms": 67.263, "count": 98, "max_ms": 92.202, "p95_ms": 88.491}, "visualize": {"avg_ms": 11.589, "count": 98, "max_ms": 14.393, "p95_ms": 12.836}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 555}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4041.145, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5626cee84340] error while decoding MB 64 27, bytestream -20", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 118, "h264_decode_error_count": 123, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:20:35+09:00", "last_error_summary": "[h264 @ 0x5626cee84340] error while decoding MB 64 27, bytestream -20", "last_exit_at": "2026-07-01T19:18:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.522, "last_started_at": "2026-07-01T19:18:45+09:00", "pid": 18469, "process": {"exists": false, "pid": 18469}, "restart_count": 138, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 139}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.741, "sample_count": 1148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 66665, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:20:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T19:20:35+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T19:20:35+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T19:20:35.112052+09:00", "mtime_epoch": 1782901235.1120522, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186069}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:20:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.008, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.095, "pushed": 4.095, "read": 4.095}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.29, "count": 122, "max_ms": 12.867, "p95_ms": 10.351}, "gstreamer_push": {"avg_ms": 0.933, "count": 122, "max_ms": 2.897, "p95_ms": 1.76}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.125, "count": 122, "max_ms": 70.975, "p95_ms": 62.974}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.352, "count": 122, "max_ms": 91.938, "p95_ms": 87.497}, "yolo11_640_fire_11class": {"avg_ms": 53.109, "count": 122, "max_ms": 74.282, "p95_ms": 71.486}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.92, "count": 122, "max_ms": 93.698, "p95_ms": 89.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.281, "count": 122, "max_ms": 289.475, "p95_ms": 275.126}, "read_interval": {"avg_ms": 245.669, "count": 122, "max_ms": 301.142, "p95_ms": 284.876}, "triton_wait": {"avg_ms": 67.93, "count": 122, "max_ms": 93.711, "p95_ms": 90.275}, "visualize": {"avg_ms": 11.111, "count": 122, "max_ms": 15.665, "p95_ms": 12.238}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.795, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 555}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4035.059, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:20:35+09:00", "last_error_summary": "[h264 @ 0x5626cee84340] error while decoding MB 64 27, bytestream -20", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 118, "h264_decode_error_count": 123, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:20:35+09:00", "last_error_summary": "[h264 @ 0x5626cee84340] error while decoding MB 64 27, bytestream -20", "last_exit_at": "2026-07-01T19:20:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.727, "last_started_at": "2026-07-01T19:18:45+09:00", "pid": 18469, "process": {"exists": false, "pid": 18469}, "restart_count": 138, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 139}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.741, "sample_count": 1148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 66666, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:20:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T19:20:35+09:00", "last_frame_read_age_sec": 0.258, "last_frame_read_at": "2026-07-01T19:20:35+09:00", "mjpeg": {"age_sec": 0.25, "exists": true, "mtime": "2026-07-01T19:20:35.112052+09:00", "mtime_epoch": 1782901235.1120522, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186069}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:20:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.008, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.1, "pushed": 4.1, "read": 4.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.294, "count": 122, "max_ms": 12.867, "p95_ms": 10.351}, "gstreamer_push": {"avg_ms": 0.933, "count": 123, "max_ms": 2.897, "p95_ms": 1.76}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.105, "count": 122, "max_ms": 70.975, "p95_ms": 62.974}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.365, "count": 122, "max_ms": 91.938, "p95_ms": 87.497}, "yolo11_640_fire_11class": {"avg_ms": 53.14, "count": 122, "max_ms": 74.282, "p95_ms": 71.486}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.938, "count": 122, "max_ms": 93.698, "p95_ms": 89.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.413, "count": 123, "max_ms": 289.475, "p95_ms": 275.126}, "read_interval": {"avg_ms": 245.669, "count": 122, "max_ms": 301.142, "p95_ms": 284.876}, "triton_wait": {"avg_ms": 67.949, "count": 122, "max_ms": 93.711, "p95_ms": 90.275}, "visualize": {"avg_ms": 11.11, "count": 122, "max_ms": 15.665, "p95_ms": 12.227}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 555}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4035.051, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 118, "h264_decode_error_count": 123, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:20:35+09:00", "last_error_summary": "[h264 @ 0x5626cee84340] error while decoding MB 64 27, bytestream -20", "last_exit_at": "2026-07-01T19:20:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.73, "last_started_at": "2026-07-01T19:18:45+09:00", "pid": 18469, "process": {"exists": false, "pid": 18469}, "restart_count": 138, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 139}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.741, "sample_count": 1148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 66666, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:20:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T19:20:35+09:00", "last_frame_read_age_sec": 0.261, "last_frame_read_at": "2026-07-01T19:20:35+09:00", "mjpeg": {"age_sec": 0.253, "exists": true, "mtime": "2026-07-01T19:20:35.112052+09:00", "mtime_epoch": 1782901235.1120522, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186069}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:20:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.008, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.068, "pushed": 4.068, "read": 4.068}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.294, "count": 122, "max_ms": 12.867, "p95_ms": 10.351}, "gstreamer_push": {"avg_ms": 0.933, "count": 122, "max_ms": 2.897, "p95_ms": 1.76}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.105, "count": 122, "max_ms": 70.975, "p95_ms": 62.974}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.365, "count": 122, "max_ms": 91.938, "p95_ms": 87.497}, "yolo11_640_fire_11class": {"avg_ms": 53.14, "count": 122, "max_ms": 74.282, "p95_ms": 71.486}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.938, "count": 122, "max_ms": 93.698, "p95_ms": 89.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.463, "count": 122, "max_ms": 289.475, "p95_ms": 275.126}, "read_interval": {"avg_ms": 245.669, "count": 122, "max_ms": 301.142, "p95_ms": 284.876}, "triton_wait": {"avg_ms": 67.949, "count": 122, "max_ms": 93.711, "p95_ms": 90.275}, "visualize": {"avg_ms": 11.11, "count": 122, "max_ms": 15.665, "p95_ms": 12.227}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 555}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4145.203, "limit_mb": 8796093022207.996}, "detail": {"pid": 18576, "restart_count": 139, "start_count": 140}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 118, "h264_decode_error_count": 123, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:20:35+09:00", "last_error_summary": "[h264 @ 0x5626cee84340] error while decoding MB 64 27, bytestream -20", "last_exit_at": "2026-07-01T19:20:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:20:38+09:00", "pid": 18576, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.309, "name": "ffmpeg", "pid": 18576, "rss_mb": 139.309, "state": "S (sleeping)", "threads": 106}, "restart_count": 139, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 140}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.839, "sample_count": 1128, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 66666, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:20:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 5.019, "last_frame_pushed_at": "2026-07-01T19:20:35+09:00", "last_frame_read_age_sec": 5.273, "last_frame_read_at": "2026-07-01T19:20:35+09:00", "mjpeg": {"age_sec": 5.005, "exists": true, "mtime": "2026-07-01T19:20:35.372056+09:00", "mtime_epoch": 1782901235.372056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183981}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:20:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.008, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.435, "pushed": 3.435, "read": 3.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.205, "count": 102, "max_ms": 12.867, "p95_ms": 10.343}, "gstreamer_push": {"avg_ms": 0.945, "count": 103, "max_ms": 2.897, "p95_ms": 1.813}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.602, "count": 102, "max_ms": 70.975, "p95_ms": 61.206}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.853, "count": 102, "max_ms": 91.938, "p95_ms": 85.578}, "yolo11_640_fire_11class": {"avg_ms": 52.71, "count": 102, "max_ms": 72.379, "p95_ms": 70.72}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.253, "count": 102, "max_ms": 91.862, "p95_ms": 87.675}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.591, "count": 103, "max_ms": 289.475, "p95_ms": 275.771}, "read_interval": {"avg_ms": 244.807, "count": 102, "max_ms": 301.142, "p95_ms": 286.139}, "triton_wait": {"avg_ms": 67.362, "count": 102, "max_ms": 91.95, "p95_ms": 88.483}, "visualize": {"avg_ms": 11.07, "count": 102, "max_ms": 15.665, "p95_ms": 12.145}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 559}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4038.902, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a6c0a78d40] error while decoding MB 75 7, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 119, "h264_decode_error_count": 124, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:22:33+09:00", "last_error_summary": "[h264 @ 0x55a6c0a78d40] error while decoding MB 75 7, bytestream -5", "last_exit_at": "2026-07-01T19:20:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.295, "last_started_at": "2026-07-01T19:20:38+09:00", "pid": 18576, "process": {"exists": false, "pid": 18576}, "restart_count": 139, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 140}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.736, "sample_count": 1135, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67101, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:22:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T19:22:33+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T19:22:33+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T19:22:33.625741+09:00", "mtime_epoch": 1782901353.6257415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186909}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:22:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 890.734, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.925, "pushed": 3.925, "read": 3.925}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.018, "count": 117, "max_ms": 11.801, "p95_ms": 10.702}, "gstreamer_push": {"avg_ms": 0.901, "count": 117, "max_ms": 2.231, "p95_ms": 0.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.173, "count": 117, "max_ms": 65.587, "p95_ms": 60.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.524, "count": 117, "max_ms": 93.135, "p95_ms": 90.112}, "yolo11_640_fire_11class": {"avg_ms": 52.231, "count": 117, "max_ms": 73.773, "p95_ms": 71.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.005, "count": 117, "max_ms": 92.165, "p95_ms": 89.977}}, "truncated_count": 0}, "processing_total": {"avg_ms": 246.955, "count": 117, "max_ms": 325.921, "p95_ms": 273.842}, "read_interval": {"avg_ms": 256.567, "count": 117, "max_ms": 335.974, "p95_ms": 283.209}, "triton_wait": {"avg_ms": 67.489, "count": 117, "max_ms": 93.147, "p95_ms": 90.757}, "visualize": {"avg_ms": 11.69, "count": 117, "max_ms": 14.357, "p95_ms": 12.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.81, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 559}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4038.34, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:22:33+09:00", "last_error_summary": "[h264 @ 0x55a6c0a78d40] error while decoding MB 75 7, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 119, "h264_decode_error_count": 124, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:22:33+09:00", "last_error_summary": "[h264 @ 0x55a6c0a78d40] error while decoding MB 75 7, bytestream -5", "last_exit_at": "2026-07-01T19:22:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.488, "last_started_at": "2026-07-01T19:20:38+09:00", "pid": 18576, "process": {"exists": false, "pid": 18576}, "restart_count": 139, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 140}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.736, "sample_count": 1136, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67102, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:22:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:22:33+09:00", "last_frame_read_age_sec": 0.248, "last_frame_read_at": "2026-07-01T19:22:33+09:00", "mjpeg": {"age_sec": 0.239, "exists": true, "mtime": "2026-07-01T19:22:33.625741+09:00", "mtime_epoch": 1782901353.6257415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186909}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:22:33+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 890.734, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.901, "pushed": 3.901, "read": 3.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.018, "count": 117, "max_ms": 11.801, "p95_ms": 10.702}, "gstreamer_push": {"avg_ms": 0.901, "count": 117, "max_ms": 2.231, "p95_ms": 0.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.959, "count": 117, "max_ms": 65.587, "p95_ms": 60.479}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.474, "count": 117, "max_ms": 93.135, "p95_ms": 90.112}, "yolo11_640_fire_11class": {"avg_ms": 52.182, "count": 117, "max_ms": 73.773, "p95_ms": 71.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.917, "count": 117, "max_ms": 92.165, "p95_ms": 89.977}}, "truncated_count": 0}, "processing_total": {"avg_ms": 246.793, "count": 117, "max_ms": 325.921, "p95_ms": 273.842}, "read_interval": {"avg_ms": 256.567, "count": 117, "max_ms": 335.974, "p95_ms": 283.209}, "triton_wait": {"avg_ms": 67.4, "count": 117, "max_ms": 93.147, "p95_ms": 90.757}, "visualize": {"avg_ms": 11.685, "count": 117, "max_ms": 14.357, "p95_ms": 12.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 559}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4038.332, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 119, "h264_decode_error_count": 124, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:22:33+09:00", "last_error_summary": "[h264 @ 0x55a6c0a78d40] error while decoding MB 75 7, bytestream -5", "last_exit_at": "2026-07-01T19:22:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 115.492, "last_started_at": "2026-07-01T19:20:38+09:00", "pid": 18576, "process": {"exists": false, "pid": 18576}, "restart_count": 139, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 140}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.736, "sample_count": 1136, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67102, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:22:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:22:33+09:00", "last_frame_read_age_sec": 0.252, "last_frame_read_at": "2026-07-01T19:22:33+09:00", "mjpeg": {"age_sec": 0.242, "exists": true, "mtime": "2026-07-01T19:22:33.625741+09:00", "mtime_epoch": 1782901353.6257415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186909}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:22:33+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 890.734, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.9, "pushed": 3.9, "read": 3.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.018, "count": 117, "max_ms": 11.801, "p95_ms": 10.702}, "gstreamer_push": {"avg_ms": 0.901, "count": 117, "max_ms": 2.231, "p95_ms": 0.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.959, "count": 117, "max_ms": 65.587, "p95_ms": 60.479}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.474, "count": 117, "max_ms": 93.135, "p95_ms": 90.112}, "yolo11_640_fire_11class": {"avg_ms": 52.182, "count": 117, "max_ms": 73.773, "p95_ms": 71.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.917, "count": 117, "max_ms": 92.165, "p95_ms": 89.977}}, "truncated_count": 0}, "processing_total": {"avg_ms": 246.793, "count": 117, "max_ms": 325.921, "p95_ms": 273.842}, "read_interval": {"avg_ms": 256.567, "count": 117, "max_ms": 335.974, "p95_ms": 283.209}, "triton_wait": {"avg_ms": 67.4, "count": 117, "max_ms": 93.147, "p95_ms": 90.757}, "visualize": {"avg_ms": 11.685, "count": 117, "max_ms": 14.357, "p95_ms": 12.491}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 559}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4146.898, "limit_mb": 8796093022207.996}, "detail": {"pid": 18683, "restart_count": 140, "start_count": 141}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 119, "h264_decode_error_count": 124, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:22:33+09:00", "last_error_summary": "[h264 @ 0x55a6c0a78d40] error while decoding MB 75 7, bytestream -5", "last_exit_at": "2026-07-01T19:22:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:22:36+09:00", "pid": 18683, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.754, "name": "ffmpeg", "pid": 18683, "rss_mb": 139.754, "state": "S (sleeping)", "threads": 106}, "restart_count": 140, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.736, "sample_count": 1119, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67102, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:22:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.287, "last_frame_pushed_at": "2026-07-01T19:22:33+09:00", "last_frame_read_age_sec": 4.533, "last_frame_read_at": "2026-07-01T19:22:33+09:00", "mjpeg": {"age_sec": 4.272, "exists": true, "mtime": "2026-07-01T19:22:33.877745+09:00", "mtime_epoch": 1782901353.8777452, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178823}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:22:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 890.734, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.384, "pushed": 3.384, "read": 3.35}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.073, "count": 100, "max_ms": 11.801, "p95_ms": 10.702}, "gstreamer_push": {"avg_ms": 0.873, "count": 101, "max_ms": 1.859, "p95_ms": 0.883}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.978, "count": 100, "max_ms": 65.587, "p95_ms": 60.219}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.136, "count": 100, "max_ms": 93.135, "p95_ms": 89.946}, "yolo11_640_fire_11class": {"avg_ms": 51.935, "count": 100, "max_ms": 73.773, "p95_ms": 70.036}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.722, "count": 100, "max_ms": 92.165, "p95_ms": 89.591}}, "truncated_count": 0}, "processing_total": {"avg_ms": 245.992, "count": 101, "max_ms": 319.095, "p95_ms": 272.959}, "read_interval": {"avg_ms": 255.595, "count": 100, "max_ms": 329.974, "p95_ms": 282.148}, "triton_wait": {"avg_ms": 67.079, "count": 100, "max_ms": 93.147, "p95_ms": 90.463}, "visualize": {"avg_ms": 11.692, "count": 100, "max_ms": 14.357, "p95_ms": 12.396}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.849, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 562}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4037.266, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56052f408240] error while decoding MB 35 32, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 120, "h264_decode_error_count": 125, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:24:28+09:00", "last_error_summary": "[h264 @ 0x56052f408240] error while decoding MB 35 32, bytestream -9", "last_exit_at": "2026-07-01T19:22:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.824, "last_started_at": "2026-07-01T19:22:36+09:00", "pid": 18683, "process": {"exists": false, "pid": 18683}, "restart_count": 140, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.408, "sample_count": 1132, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67528, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:24:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T19:24:28+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T19:24:28+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T19:24:28.931405+09:00", "mtime_epoch": 1782901468.931405, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183094}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:24:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.445, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.882, "pushed": 3.882, "read": 3.882}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.005, "count": 115, "max_ms": 13.844, "p95_ms": 11.089}, "gstreamer_push": {"avg_ms": 0.915, "count": 116, "max_ms": 2.925, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.81, "count": 115, "max_ms": 70.773, "p95_ms": 60.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.995, "count": 115, "max_ms": 92.919, "p95_ms": 88.526}, "yolo11_640_fire_11class": {"avg_ms": 53.215, "count": 115, "max_ms": 73.534, "p95_ms": 69.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.106, "count": 115, "max_ms": 91.383, "p95_ms": 88.933}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.635, "count": 116, "max_ms": 341.172, "p95_ms": 272.168}, "read_interval": {"avg_ms": 259.28, "count": 116, "max_ms": 350.345, "p95_ms": 282.228}, "triton_wait": {"avg_ms": 68.947, "count": 115, "max_ms": 92.931, "p95_ms": 89.145}, "visualize": {"avg_ms": 11.671, "count": 115, "max_ms": 15.905, "p95_ms": 12.939}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 562}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4036.148, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:24:28+09:00", "last_error_summary": "[h264 @ 0x56052f408240] error while decoding MB 35 32, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 120, "h264_decode_error_count": 125, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:24:28+09:00", "last_error_summary": "[h264 @ 0x56052f408240] error while decoding MB 35 32, bytestream -9", "last_exit_at": "2026-07-01T19:24:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.056, "last_started_at": "2026-07-01T19:22:36+09:00", "pid": 18683, "process": {"exists": false, "pid": 18683}, "restart_count": 140, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.416, "sample_count": 1132, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67529, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:24:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:24:29+09:00", "last_frame_read_age_sec": 0.283, "last_frame_read_at": "2026-07-01T19:24:28+09:00", "mjpeg": {"age_sec": 0.273, "exists": true, "mtime": "2026-07-01T19:24:28.931405+09:00", "mtime_epoch": 1782901468.931405, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183094}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:24:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.445, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.884, "pushed": 3.884, "read": 3.851}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.005, "count": 115, "max_ms": 13.844, "p95_ms": 11.089}, "gstreamer_push": {"avg_ms": 0.914, "count": 116, "max_ms": 2.925, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.969, "count": 115, "max_ms": 70.773, "p95_ms": 60.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.165, "count": 115, "max_ms": 92.919, "p95_ms": 88.526}, "yolo11_640_fire_11class": {"avg_ms": 53.419, "count": 115, "max_ms": 73.534, "p95_ms": 69.804}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.38, "count": 115, "max_ms": 91.383, "p95_ms": 89.132}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.911, "count": 116, "max_ms": 341.172, "p95_ms": 273.632}, "read_interval": {"avg_ms": 259.312, "count": 115, "max_ms": 350.345, "p95_ms": 282.228}, "triton_wait": {"avg_ms": 69.221, "count": 115, "max_ms": 92.931, "p95_ms": 89.436}, "visualize": {"avg_ms": 11.67, "count": 115, "max_ms": 15.905, "p95_ms": 12.939}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.863, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 562}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4036.141, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 120, "h264_decode_error_count": 125, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:24:28+09:00", "last_error_summary": "[h264 @ 0x56052f408240] error while decoding MB 35 32, bytestream -9", "last_exit_at": "2026-07-01T19:24:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.06, "last_started_at": "2026-07-01T19:22:36+09:00", "pid": 18683, "process": {"exists": false, "pid": 18683}, "restart_count": 140, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 141}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.416, "sample_count": 1132, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67529, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:24:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:24:29+09:00", "last_frame_read_age_sec": 0.286, "last_frame_read_at": "2026-07-01T19:24:28+09:00", "mjpeg": {"age_sec": 0.277, "exists": true, "mtime": "2026-07-01T19:24:28.931405+09:00", "mtime_epoch": 1782901468.931405, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183094}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:24:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.445, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.884, "pushed": 3.884, "read": 3.851}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.005, "count": 115, "max_ms": 13.844, "p95_ms": 11.089}, "gstreamer_push": {"avg_ms": 0.914, "count": 116, "max_ms": 2.925, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.969, "count": 115, "max_ms": 70.773, "p95_ms": 60.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.165, "count": 115, "max_ms": 92.919, "p95_ms": 88.526}, "yolo11_640_fire_11class": {"avg_ms": 53.419, "count": 115, "max_ms": 73.534, "p95_ms": 69.804}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.38, "count": 115, "max_ms": 91.383, "p95_ms": 89.132}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.911, "count": 116, "max_ms": 341.172, "p95_ms": 273.632}, "read_interval": {"avg_ms": 259.312, "count": 115, "max_ms": 350.345, "p95_ms": 282.228}, "triton_wait": {"avg_ms": 69.221, "count": 115, "max_ms": 92.931, "p95_ms": 89.436}, "visualize": {"avg_ms": 11.67, "count": 115, "max_ms": 15.905, "p95_ms": 12.939}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 562}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4149.758, "limit_mb": 8796093022207.996}, "detail": {"pid": 18790, "restart_count": 141, "start_count": 142}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 120, "h264_decode_error_count": 125, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:24:28+09:00", "last_error_summary": "[h264 @ 0x56052f408240] error while decoding MB 35 32, bytestream -9", "last_exit_at": "2026-07-01T19:24:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:24:32+09:00", "pid": 18790, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.117, "name": "ffmpeg", "pid": 18790, "rss_mb": 139.117, "state": "S (sleeping)", "threads": 106}, "restart_count": 141, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.422, "sample_count": 1112, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67529, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:24:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.866, "last_frame_pushed_at": "2026-07-01T19:24:29+09:00", "last_frame_read_age_sec": 5.146, "last_frame_read_at": "2026-07-01T19:24:28+09:00", "mjpeg": {"age_sec": 4.848, "exists": true, "mtime": "2026-07-01T19:24:29.219409+09:00", "mtime_epoch": 1782901469.2194092, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183535}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:24:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.445, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.276, "pushed": 3.276, "read": 3.243}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.953, "count": 97, "max_ms": 13.844, "p95_ms": 11.13}, "gstreamer_push": {"avg_ms": 0.926, "count": 98, "max_ms": 2.925, "p95_ms": 0.957}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.92, "count": 97, "max_ms": 70.773, "p95_ms": 60.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.252, "count": 97, "max_ms": 90.136, "p95_ms": 85.441}, "yolo11_640_fire_11class": {"avg_ms": 52.924, "count": 97, "max_ms": 70.93, "p95_ms": 69.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.864, "count": 97, "max_ms": 91.362, "p95_ms": 88.933}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.545, "count": 98, "max_ms": 329.753, "p95_ms": 273.632}, "read_interval": {"avg_ms": 257.91, "count": 97, "max_ms": 339.098, "p95_ms": 280.205}, "triton_wait": {"avg_ms": 68.43, "count": 97, "max_ms": 91.373, "p95_ms": 89.145}, "visualize": {"avg_ms": 11.695, "count": 97, "max_ms": 15.905, "p95_ms": 12.95}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 566}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4040.539, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5603a98c67c0] error while decoding MB 0 35, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 121, "h264_decode_error_count": 126, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:26:22+09:00", "last_error_summary": "[h264 @ 0x5603a98c67c0] error while decoding MB 0 35, bytestream -7", "last_exit_at": "2026-07-01T19:24:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.701, "last_started_at": "2026-07-01T19:24:32+09:00", "pid": 18790, "process": {"exists": false, "pid": 18790}, "restart_count": 141, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.066, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67949, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:26:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T19:26:22+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T19:26:22+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T19:26:22.725062+09:00", "mtime_epoch": 1782901582.7250624, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186034}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:26:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 892.062, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.992, "pushed": 3.992, "read": 3.992}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.707, "count": 119, "max_ms": 13.161, "p95_ms": 10.895}, "gstreamer_push": {"avg_ms": 0.932, "count": 119, "max_ms": 2.822, "p95_ms": 1.256}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.943, "count": 119, "max_ms": 68.959, "p95_ms": 57.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.788, "count": 119, "max_ms": 89.519, "p95_ms": 86.327}, "yolo11_640_fire_11class": {"avg_ms": 51.319, "count": 119, "max_ms": 71.308, "p95_ms": 66.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.9, "count": 119, "max_ms": 91.275, "p95_ms": 85.887}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.418, "count": 119, "max_ms": 320.964, "p95_ms": 272.485}, "read_interval": {"avg_ms": 252.126, "count": 119, "max_ms": 330.299, "p95_ms": 281.492}, "triton_wait": {"avg_ms": 66.63, "count": 119, "max_ms": 91.287, "p95_ms": 86.723}, "visualize": {"avg_ms": 11.464, "count": 119, "max_ms": 14.173, "p95_ms": 12.484}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.806, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 566}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4039.438, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:26:22+09:00", "last_error_summary": "[h264 @ 0x5603a98c67c0] error while decoding MB 0 35, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 121, "h264_decode_error_count": 126, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:26:22+09:00", "last_error_summary": "[h264 @ 0x5603a98c67c0] error while decoding MB 0 35, bytestream -7", "last_exit_at": "2026-07-01T19:26:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.904, "last_started_at": "2026-07-01T19:24:32+09:00", "pid": 18790, "process": {"exists": false, "pid": 18790}, "restart_count": 141, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.066, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67950, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:26:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T19:26:22+09:00", "last_frame_read_age_sec": 0.253, "last_frame_read_at": "2026-07-01T19:26:22+09:00", "mjpeg": {"age_sec": 0.246, "exists": true, "mtime": "2026-07-01T19:26:22.725062+09:00", "mtime_epoch": 1782901582.7250624, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186034}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:26:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 892.062, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.967, "pushed": 3.967, "read": 3.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.708, "count": 119, "max_ms": 13.161, "p95_ms": 10.895}, "gstreamer_push": {"avg_ms": 0.931, "count": 119, "max_ms": 2.822, "p95_ms": 1.256}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.985, "count": 119, "max_ms": 68.959, "p95_ms": 57.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.765, "count": 119, "max_ms": 89.519, "p95_ms": 86.327}, "yolo11_640_fire_11class": {"avg_ms": 51.275, "count": 119, "max_ms": 71.308, "p95_ms": 66.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.872, "count": 119, "max_ms": 91.275, "p95_ms": 85.887}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.42, "count": 119, "max_ms": 320.964, "p95_ms": 272.485}, "read_interval": {"avg_ms": 252.126, "count": 119, "max_ms": 330.299, "p95_ms": 281.492}, "triton_wait": {"avg_ms": 66.607, "count": 119, "max_ms": 91.287, "p95_ms": 86.723}, "visualize": {"avg_ms": 11.465, "count": 119, "max_ms": 14.173, "p95_ms": 12.484}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 566}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4039.43, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 121, "h264_decode_error_count": 126, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:26:22+09:00", "last_error_summary": "[h264 @ 0x5603a98c67c0] error while decoding MB 0 35, bytestream -7", "last_exit_at": "2026-07-01T19:26:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.907, "last_started_at": "2026-07-01T19:24:32+09:00", "pid": 18790, "process": {"exists": false, "pid": 18790}, "restart_count": 141, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 142}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.066, "sample_count": 1122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67950, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:26:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T19:26:22+09:00", "last_frame_read_age_sec": 0.257, "last_frame_read_at": "2026-07-01T19:26:22+09:00", "mjpeg": {"age_sec": 0.249, "exists": true, "mtime": "2026-07-01T19:26:22.725062+09:00", "mtime_epoch": 1782901582.7250624, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186034}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:26:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 892.062, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.999, "pushed": 3.999, "read": 3.965}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.708, "count": 119, "max_ms": 13.161, "p95_ms": 10.895}, "gstreamer_push": {"avg_ms": 0.931, "count": 119, "max_ms": 2.822, "p95_ms": 1.256}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.985, "count": 119, "max_ms": 68.959, "p95_ms": 57.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.765, "count": 119, "max_ms": 89.519, "p95_ms": 86.327}, "yolo11_640_fire_11class": {"avg_ms": 51.275, "count": 119, "max_ms": 71.308, "p95_ms": 66.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.872, "count": 119, "max_ms": 91.275, "p95_ms": 85.887}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.42, "count": 119, "max_ms": 320.964, "p95_ms": 272.485}, "read_interval": {"avg_ms": 252.085, "count": 118, "max_ms": 330.299, "p95_ms": 281.492}, "triton_wait": {"avg_ms": 66.607, "count": 119, "max_ms": 91.287, "p95_ms": 86.723}, "visualize": {"avg_ms": 11.465, "count": 119, "max_ms": 14.173, "p95_ms": 12.484}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.757, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 566}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4147.75, "limit_mb": 8796093022207.996}, "detail": {"pid": 18897, "restart_count": 142, "start_count": 143}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 121, "h264_decode_error_count": 126, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:26:22+09:00", "last_error_summary": "[h264 @ 0x5603a98c67c0] error while decoding MB 0 35, bytestream -7", "last_exit_at": "2026-07-01T19:26:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:26:25+09:00", "pid": 18897, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.66, "name": "ffmpeg", "pid": 18897, "rss_mb": 138.66, "state": "S (sleeping)", "threads": 106}, "restart_count": 142, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.017, "sample_count": 1105, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 67950, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:26:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.689, "last_frame_pushed_at": "2026-07-01T19:26:22+09:00", "last_frame_read_age_sec": 4.939, "last_frame_read_at": "2026-07-01T19:26:22+09:00", "mjpeg": {"age_sec": 4.675, "exists": true, "mtime": "2026-07-01T19:26:22.981066+09:00", "mtime_epoch": 1782901582.9810662, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186067}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:26:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 892.062, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.383, "pushed": 3.383, "read": 3.35}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.706, "count": 100, "max_ms": 13.161, "p95_ms": 10.851}, "gstreamer_push": {"avg_ms": 0.931, "count": 101, "max_ms": 2.822, "p95_ms": 1.167}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.212, "count": 100, "max_ms": 68.959, "p95_ms": 57.457}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.092, "count": 100, "max_ms": 89.519, "p95_ms": 85.773}, "yolo11_640_fire_11class": {"avg_ms": 51.629, "count": 100, "max_ms": 71.308, "p95_ms": 65.497}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.247, "count": 100, "max_ms": 91.275, "p95_ms": 85.558}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.004, "count": 101, "max_ms": 320.964, "p95_ms": 270.605}, "read_interval": {"avg_ms": 251.657, "count": 100, "max_ms": 330.299, "p95_ms": 279.679}, "triton_wait": {"avg_ms": 66.987, "count": 100, "max_ms": 91.287, "p95_ms": 86.47}, "visualize": {"avg_ms": 11.442, "count": 100, "max_ms": 14.173, "p95_ms": 12.481}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.853, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4038.988, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ee7db45180] error while decoding MB 44 33, bytestream -28", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 122, "h264_decode_error_count": 127, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:28:15+09:00", "last_error_summary": "[h264 @ 0x55ee7db45180] error while decoding MB 44 33, bytestream -28", "last_exit_at": "2026-07-01T19:26:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.138, "last_started_at": "2026-07-01T19:26:25+09:00", "pid": 18897, "process": {"exists": false, "pid": 18897}, "restart_count": 142, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.383, "sample_count": 1123, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68367, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:28:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T19:28:15+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T19:28:15+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T19:28:15.750721+09:00", "mtime_epoch": 1782901695.7507207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187764}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:28:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.926, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.855, "pushed": 3.855, "read": 3.855}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.948, "count": 115, "max_ms": 14.786, "p95_ms": 10.783}, "gstreamer_push": {"avg_ms": 0.905, "count": 115, "max_ms": 3.019, "p95_ms": 0.923}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.741, "count": 115, "max_ms": 70.356, "p95_ms": 65.69}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.87, "count": 115, "max_ms": 94.73, "p95_ms": 91.592}, "yolo11_640_fire_11class": {"avg_ms": 53.563, "count": 115, "max_ms": 75.257, "p95_ms": 69.273}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.347, "count": 115, "max_ms": 91.991, "p95_ms": 90.669}}, "truncated_count": 0}, "processing_total": {"avg_ms": 251.408, "count": 115, "max_ms": 335.778, "p95_ms": 277.029}, "read_interval": {"avg_ms": 260.916, "count": 115, "max_ms": 347.497, "p95_ms": 286.731}, "triton_wait": {"avg_ms": 70.745, "count": 115, "max_ms": 94.742, "p95_ms": 92.003}, "visualize": {"avg_ms": 11.78, "count": 115, "max_ms": 17.09, "p95_ms": 12.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.834, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4037.934, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:28:15+09:00", "last_error_summary": "[h264 @ 0x55ee7db45180] error while decoding MB 44 33, bytestream -28", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 122, "h264_decode_error_count": 127, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:28:15+09:00", "last_error_summary": "[h264 @ 0x55ee7db45180] error while decoding MB 44 33, bytestream -28", "last_exit_at": "2026-07-01T19:28:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.356, "last_started_at": "2026-07-01T19:26:25+09:00", "pid": 18897, "process": {"exists": false, "pid": 18897}, "restart_count": 142, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.3, "sample_count": 1123, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68368, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:28:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:28:16+09:00", "last_frame_read_age_sec": 0.271, "last_frame_read_at": "2026-07-01T19:28:15+09:00", "mjpeg": {"age_sec": 0.26, "exists": true, "mtime": "2026-07-01T19:28:15.750721+09:00", "mtime_epoch": 1782901695.7507207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187764}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:28:16+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.926, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.857, "pushed": 3.857, "read": 3.824}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.952, "count": 115, "max_ms": 14.786, "p95_ms": 10.783}, "gstreamer_push": {"avg_ms": 0.905, "count": 115, "max_ms": 3.019, "p95_ms": 0.923}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.725, "count": 115, "max_ms": 70.356, "p95_ms": 65.69}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.084, "count": 115, "max_ms": 94.73, "p95_ms": 91.592}, "yolo11_640_fire_11class": {"avg_ms": 53.719, "count": 115, "max_ms": 75.257, "p95_ms": 69.273}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.566, "count": 115, "max_ms": 91.991, "p95_ms": 90.669}}, "truncated_count": 0}, "processing_total": {"avg_ms": 251.765, "count": 115, "max_ms": 335.778, "p95_ms": 277.029}, "read_interval": {"avg_ms": 261.134, "count": 114, "max_ms": 347.497, "p95_ms": 286.731}, "triton_wait": {"avg_ms": 70.959, "count": 115, "max_ms": 94.742, "p95_ms": 92.003}, "visualize": {"avg_ms": 11.784, "count": 115, "max_ms": 17.09, "p95_ms": 12.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.814, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4037.926, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 122, "h264_decode_error_count": 127, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:28:15+09:00", "last_error_summary": "[h264 @ 0x55ee7db45180] error while decoding MB 44 33, bytestream -28", "last_exit_at": "2026-07-01T19:28:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.359, "last_started_at": "2026-07-01T19:26:25+09:00", "pid": 18897, "process": {"exists": false, "pid": 18897}, "restart_count": 142, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 143}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.3, "sample_count": 1123, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68368, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:28:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:28:16+09:00", "last_frame_read_age_sec": 0.274, "last_frame_read_at": "2026-07-01T19:28:15+09:00", "mjpeg": {"age_sec": 0.263, "exists": true, "mtime": "2026-07-01T19:28:15.750721+09:00", "mtime_epoch": 1782901695.7507207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187764}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:28:16+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.926, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.857, "pushed": 3.857, "read": 3.823}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.952, "count": 115, "max_ms": 14.786, "p95_ms": 10.783}, "gstreamer_push": {"avg_ms": 0.905, "count": 115, "max_ms": 3.019, "p95_ms": 0.923}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.725, "count": 115, "max_ms": 70.356, "p95_ms": 65.69}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.084, "count": 115, "max_ms": 94.73, "p95_ms": 91.592}, "yolo11_640_fire_11class": {"avg_ms": 53.719, "count": 115, "max_ms": 75.257, "p95_ms": 69.273}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.566, "count": 115, "max_ms": 91.991, "p95_ms": 90.669}}, "truncated_count": 0}, "processing_total": {"avg_ms": 251.765, "count": 115, "max_ms": 335.778, "p95_ms": 277.029}, "read_interval": {"avg_ms": 261.134, "count": 114, "max_ms": 347.497, "p95_ms": 286.731}, "triton_wait": {"avg_ms": 70.959, "count": 115, "max_ms": 94.742, "p95_ms": 92.003}, "visualize": {"avg_ms": 11.784, "count": 115, "max_ms": 17.09, "p95_ms": 12.785}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.817, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 569}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4145.926, "limit_mb": 8796093022207.996}, "detail": {"pid": 19004, "restart_count": 143, "start_count": 144}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 122, "h264_decode_error_count": 127, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:28:15+09:00", "last_error_summary": "[h264 @ 0x55ee7db45180] error while decoding MB 44 33, bytestream -28", "last_exit_at": "2026-07-01T19:28:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:28:18+09:00", "pid": 19004, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.625, "name": "ffmpeg", "pid": 19004, "rss_mb": 136.625, "state": "S (sleeping)", "threads": 106}, "restart_count": 143, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 144}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.383, "sample_count": 1106, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68368, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:28:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.254, "last_frame_pushed_at": "2026-07-01T19:28:16+09:00", "last_frame_read_age_sec": 4.522, "last_frame_read_at": "2026-07-01T19:28:15+09:00", "mjpeg": {"age_sec": 4.235, "exists": true, "mtime": "2026-07-01T19:28:16.026725+09:00", "mtime_epoch": 1782901696.0267248, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187809}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:28:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.496, "name": "celery", "pid": 237, "rss_mb": 891.926, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.283, "pushed": 3.283, "read": 3.249}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.968, "count": 97, "max_ms": 14.786, "p95_ms": 10.802}, "gstreamer_push": {"avg_ms": 0.901, "count": 98, "max_ms": 3.019, "p95_ms": 0.923}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.481, "count": 97, "max_ms": 70.356, "p95_ms": 65.69}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.164, "count": 97, "max_ms": 94.73, "p95_ms": 92.003}, "yolo11_640_fire_11class": {"avg_ms": 53.692, "count": 97, "max_ms": 75.257, "p95_ms": 69.344}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.481, "count": 97, "max_ms": 91.991, "p95_ms": 90.669}}, "truncated_count": 0}, "processing_total": {"avg_ms": 254.378, "count": 98, "max_ms": 335.778, "p95_ms": 277.783}, "read_interval": {"avg_ms": 263.717, "count": 97, "max_ms": 347.497, "p95_ms": 286.844}, "triton_wait": {"avg_ms": 71.035, "count": 97, "max_ms": 94.742, "p95_ms": 92.012}, "visualize": {"avg_ms": 11.933, "count": 97, "max_ms": 17.09, "p95_ms": 13.559}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.855, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 573}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4047.898, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 128, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "last_exit_at": "2026-07-01T19:28:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.95, "last_started_at": "2026-07-01T19:28:18+09:00", "pid": 19004, "process": {"exists": false, "pid": 19004}, "restart_count": 143, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 144}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.95, "sample_count": 1125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68788, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:30:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T19:30:10+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T19:30:10+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T19:30:10.168668+09:00", "mtime_epoch": 1782901810.1686683, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186912}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:30:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.777, "name": "celery", "pid": 237, "rss_mb": 897.359, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.875, "pushed": 3.875, "read": 3.875}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.224, "count": 115, "max_ms": 18.538, "p95_ms": 11.628}, "gstreamer_push": {"avg_ms": 0.944, "count": 115, "max_ms": 2.956, "p95_ms": 1.776}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.913, "count": 115, "max_ms": 72.87, "p95_ms": 64.522}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.116, "count": 115, "max_ms": 93.696, "p95_ms": 89.937}, "yolo11_640_fire_11class": {"avg_ms": 51.882, "count": 115, "max_ms": 71.999, "p95_ms": 69.425}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.907, "count": 115, "max_ms": 92.308, "p95_ms": 91.011}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.895, "count": 115, "max_ms": 361.912, "p95_ms": 292.373}, "read_interval": {"avg_ms": 260.567, "count": 115, "max_ms": 375.185, "p95_ms": 301.63}, "triton_wait": {"avg_ms": 67.734, "count": 115, "max_ms": 93.708, "p95_ms": 91.748}, "visualize": {"avg_ms": 11.929, "count": 115, "max_ms": 16.216, "p95_ms": 13.602}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.677, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 573}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4047.152, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 128, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "last_exit_at": "2026-07-01T19:30:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.138, "last_started_at": "2026-07-01T19:28:18+09:00", "pid": 19004, "process": {"exists": false, "pid": 19004}, "restart_count": 143, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 144}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.95, "sample_count": 1125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:30:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T19:30:10+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T19:30:10+09:00", "mjpeg": {"age_sec": 0.23, "exists": true, "mtime": "2026-07-01T19:30:10.168668+09:00", "mtime_epoch": 1782901810.1686683, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186912}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:30:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.777, "name": "celery", "pid": 237, "rss_mb": 897.359, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.884, "pushed": 3.884, "read": 3.851}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.217, "count": 115, "max_ms": 18.538, "p95_ms": 11.628}, "gstreamer_push": {"avg_ms": 0.943, "count": 116, "max_ms": 2.956, "p95_ms": 1.776}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.913, "count": 115, "max_ms": 72.87, "p95_ms": 64.522}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.923, "count": 115, "max_ms": 93.696, "p95_ms": 89.937}, "yolo11_640_fire_11class": {"avg_ms": 51.755, "count": 115, "max_ms": 71.999, "p95_ms": 69.159}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.732, "count": 115, "max_ms": 92.308, "p95_ms": 91.011}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.742, "count": 116, "max_ms": 361.912, "p95_ms": 292.373}, "read_interval": {"avg_ms": 260.567, "count": 115, "max_ms": 375.185, "p95_ms": 301.63}, "triton_wait": {"avg_ms": 67.55, "count": 115, "max_ms": 93.708, "p95_ms": 91.748}, "visualize": {"avg_ms": 11.929, "count": 115, "max_ms": 16.216, "p95_ms": 13.602}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.864, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 573}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4047.145, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 128, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "last_exit_at": "2026-07-01T19:30:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.141, "last_started_at": "2026-07-01T19:28:18+09:00", "pid": 19004, "process": {"exists": false, "pid": 19004}, "restart_count": 143, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 144}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.95, "sample_count": 1125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:30:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T19:30:10+09:00", "last_frame_read_age_sec": 0.241, "last_frame_read_at": "2026-07-01T19:30:10+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-01T19:30:10.168668+09:00", "mtime_epoch": 1782901810.1686683, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186912}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:30:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.777, "name": "celery", "pid": 237, "rss_mb": 897.359, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.884, "pushed": 3.884, "read": 3.85}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.217, "count": 115, "max_ms": 18.538, "p95_ms": 11.628}, "gstreamer_push": {"avg_ms": 0.943, "count": 116, "max_ms": 2.956, "p95_ms": 1.776}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.913, "count": 115, "max_ms": 72.87, "p95_ms": 64.522}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.923, "count": 115, "max_ms": 93.696, "p95_ms": 89.937}, "yolo11_640_fire_11class": {"avg_ms": 51.755, "count": 115, "max_ms": 71.999, "p95_ms": 69.159}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.732, "count": 115, "max_ms": 92.308, "p95_ms": 91.011}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.742, "count": 116, "max_ms": 361.912, "p95_ms": 292.373}, "read_interval": {"avg_ms": 260.567, "count": 115, "max_ms": 375.185, "p95_ms": 301.63}, "triton_wait": {"avg_ms": 67.55, "count": 115, "max_ms": 93.708, "p95_ms": 91.748}, "visualize": {"avg_ms": 11.929, "count": 115, "max_ms": 16.216, "p95_ms": 13.602}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 573}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4154.371, "limit_mb": 8796093022207.996}, "detail": {"pid": 19111, "restart_count": 144, "start_count": 145}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 128, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "last_exit_at": "2026-07-01T19:30:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:30:12+09:00", "pid": 19111, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.906, "name": "ffmpeg", "pid": 19111, "rss_mb": 138.906, "state": "S (sleeping)", "threads": 106}, "restart_count": 144, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 145}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.95, "sample_count": 1107, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 68789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:30:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.597, "last_frame_pushed_at": "2026-07-01T19:30:10+09:00", "last_frame_read_age_sec": 4.832, "last_frame_read_at": "2026-07-01T19:30:10+09:00", "mjpeg": {"age_sec": 4.583, "exists": true, "mtime": "2026-07-01T19:30:10.408672+09:00", "mtime_epoch": 1782901810.4086723, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186816}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:30:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.777, "name": "celery", "pid": 237, "rss_mb": 897.359, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 97, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.29, "pushed": 3.29, "read": 3.256}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.264, "count": 98, "max_ms": 18.538, "p95_ms": 11.957}, "gstreamer_push": {"avg_ms": 0.949, "count": 98, "max_ms": 2.956, "p95_ms": 1.857}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.373, "count": 98, "max_ms": 72.87, "p95_ms": 68.633}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.909, "count": 98, "max_ms": 93.696, "p95_ms": 89.937}, "yolo11_640_fire_11class": {"avg_ms": 51.735, "count": 98, "max_ms": 71.959, "p95_ms": 68.674}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.664, "count": 98, "max_ms": 92.308, "p95_ms": 90.725}}, "truncated_count": 0}, "processing_total": {"avg_ms": 249.958, "count": 98, "max_ms": 361.912, "p95_ms": 292.373}, "read_interval": {"avg_ms": 259.865, "count": 97, "max_ms": 375.185, "p95_ms": 301.63}, "triton_wait": {"avg_ms": 67.545, "count": 98, "max_ms": 93.708, "p95_ms": 91.748}, "visualize": {"avg_ms": 11.928, "count": 98, "max_ms": 16.216, "p95_ms": 13.728}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.789, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 576}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4044.746, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 128, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "last_exit_at": "2026-07-01T19:32:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.595, "last_started_at": "2026-07-01T19:30:12+09:00", "pid": 19111, "process": {"exists": false, "pid": 19111}, "restart_count": 144, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 145}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.783, "sample_count": 1130, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 69229, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:32:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:32:06+09:00", "last_frame_read_age_sec": 0.272, "last_frame_read_at": "2026-07-01T19:32:06+09:00", "mjpeg": {"age_sec": 0.263, "exists": true, "mtime": "2026-07-01T19:32:06.322608+09:00", "mtime_epoch": 1782901926.3226085, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186000}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:32:06+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.816, "name": "celery", "pid": 237, "rss_mb": 895.027, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.983, "pushed": 3.983, "read": 3.949}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.779, "count": 118, "max_ms": 13.909, "p95_ms": 11.14}, "gstreamer_push": {"avg_ms": 0.936, "count": 119, "max_ms": 2.802, "p95_ms": 1.105}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.765, "count": 118, "max_ms": 71.572, "p95_ms": 60.993}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.642, "count": 118, "max_ms": 96.932, "p95_ms": 88.927}, "yolo11_640_fire_11class": {"avg_ms": 51.935, "count": 118, "max_ms": 73.852, "p95_ms": 68.808}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.596, "count": 118, "max_ms": 94.912, "p95_ms": 88.875}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.521, "count": 119, "max_ms": 339.167, "p95_ms": 270.904}, "read_interval": {"avg_ms": 253.012, "count": 118, "max_ms": 349.162, "p95_ms": 280.8}, "triton_wait": {"avg_ms": 67.31, "count": 118, "max_ms": 96.943, "p95_ms": 89.505}, "visualize": {"avg_ms": 11.624, "count": 118, "max_ms": 15.279, "p95_ms": 13.341}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 576}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4044.738, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 128, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "last_exit_at": "2026-07-01T19:32:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 113.598, "last_started_at": "2026-07-01T19:30:12+09:00", "pid": 19111, "process": {"exists": false, "pid": 19111}, "restart_count": 144, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 145}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.783, "sample_count": 1130, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 69229, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:32:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:32:06+09:00", "last_frame_read_age_sec": 0.275, "last_frame_read_at": "2026-07-01T19:32:06+09:00", "mjpeg": {"age_sec": 0.266, "exists": true, "mtime": "2026-07-01T19:32:06.322608+09:00", "mtime_epoch": 1782901926.3226085, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186000}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:32:06+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 897.816, "name": "celery", "pid": 237, "rss_mb": 895.027, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.982, "pushed": 3.982, "read": 3.949}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.779, "count": 118, "max_ms": 13.909, "p95_ms": 11.14}, "gstreamer_push": {"avg_ms": 0.936, "count": 119, "max_ms": 2.802, "p95_ms": 1.105}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.765, "count": 118, "max_ms": 71.572, "p95_ms": 60.993}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.642, "count": 118, "max_ms": 96.932, "p95_ms": 88.927}, "yolo11_640_fire_11class": {"avg_ms": 51.935, "count": 118, "max_ms": 73.852, "p95_ms": 68.808}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.596, "count": 118, "max_ms": 94.912, "p95_ms": 88.875}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.521, "count": 119, "max_ms": 339.167, "p95_ms": 270.904}, "read_interval": {"avg_ms": 253.012, "count": 118, "max_ms": 349.162, "p95_ms": 280.8}, "triton_wait": {"avg_ms": 67.31, "count": 118, "max_ms": 96.943, "p95_ms": 89.505}, "visualize": {"avg_ms": 11.624, "count": 118, "max_ms": 15.279, "p95_ms": 13.341}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 576}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4153.043, "limit_mb": 8796093022207.996}, "detail": {"pid": 19218, "restart_count": 145, "start_count": 146}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 128, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "last_exit_at": "2026-07-01T19:32:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:32:08+09:00", "pid": 19218, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.438, "name": "ffmpeg", "pid": 19218, "rss_mb": 139.438, "state": "S (sleeping)", "threads": 106}, "restart_count": 145, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 146}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.783, "sample_count": 1113, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 69229, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:32:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.268, "last_frame_pushed_at": "2026-07-01T19:32:06+09:00", "last_frame_read_age_sec": 4.537, "last_frame_read_at": "2026-07-01T19:32:06+09:00", "mjpeg": {"age_sec": 4.252, "exists": true, "mtime": "2026-07-01T19:32:06.598613+09:00", "mtime_epoch": 1782901926.598613, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186205}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:32:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 897.816, "name": "celery", "pid": 237, "rss_mb": 895.027, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.41, "pushed": 3.41, "read": 3.376}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.757, "count": 101, "max_ms": 13.909, "p95_ms": 10.879}, "gstreamer_push": {"avg_ms": 0.943, "count": 102, "max_ms": 2.802, "p95_ms": 1.105}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.535, "count": 101, "max_ms": 71.572, "p95_ms": 60.993}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.582, "count": 101, "max_ms": 96.932, "p95_ms": 88.927}, "yolo11_640_fire_11class": {"avg_ms": 52.649, "count": 101, "max_ms": 73.852, "p95_ms": 68.808}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.467, "count": 101, "max_ms": 94.912, "p95_ms": 88.875}}, "truncated_count": 0}, "processing_total": {"avg_ms": 244.013, "count": 102, "max_ms": 339.167, "p95_ms": 270.904}, "read_interval": {"avg_ms": 253.478, "count": 101, "max_ms": 349.162, "p95_ms": 280.8}, "triton_wait": {"avg_ms": 68.258, "count": 101, "max_ms": 96.943, "p95_ms": 89.505}, "visualize": {"avg_ms": 11.599, "count": 101, "max_ms": 15.279, "p95_ms": 13.341}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 580}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4043.723, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 128, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "last_exit_at": "2026-07-01T19:33:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.574, "last_started_at": "2026-07-01T19:32:08+09:00", "pid": 19218, "process": {"exists": false, "pid": 19218}, "restart_count": 145, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 146}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.271, "sample_count": 1146, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 69658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:33:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:33:58+09:00", "last_frame_read_age_sec": 0.249, "last_frame_read_at": "2026-07-01T19:33:58+09:00", "mjpeg": {"age_sec": 0.244, "exists": true, "mtime": "2026-07-01T19:33:58.180446+09:00", "mtime_epoch": 1782902038.1804464, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183993}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:33:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 898.32, "name": "celery", "pid": 237, "rss_mb": 892.125, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.011, "pushed": 4.011, "read": 3.977}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.717, "count": 119, "max_ms": 13.549, "p95_ms": 10.951}, "gstreamer_push": {"avg_ms": 0.889, "count": 120, "max_ms": 1.841, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.574, "count": 119, "max_ms": 67.694, "p95_ms": 58.886}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.535, "count": 119, "max_ms": 95.294, "p95_ms": 90.774}, "yolo11_640_fire_11class": {"avg_ms": 51.904, "count": 119, "max_ms": 76.378, "p95_ms": 68.829}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.779, "count": 119, "max_ms": 94.215, "p95_ms": 89.469}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.595, "count": 120, "max_ms": 314.614, "p95_ms": 273.628}, "read_interval": {"avg_ms": 251.191, "count": 119, "max_ms": 324.856, "p95_ms": 286.001}, "triton_wait": {"avg_ms": 67.871, "count": 119, "max_ms": 95.31, "p95_ms": 90.787}, "visualize": {"avg_ms": 11.415, "count": 119, "max_ms": 15.514, "p95_ms": 12.687}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 580}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4043.824, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 128, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "last_exit_at": "2026-07-01T19:33:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.577, "last_started_at": "2026-07-01T19:32:08+09:00", "pid": 19218, "process": {"exists": false, "pid": 19218}, "restart_count": 145, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 146}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.271, "sample_count": 1146, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 69658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:33:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:33:58+09:00", "last_frame_read_age_sec": 0.252, "last_frame_read_at": "2026-07-01T19:33:58+09:00", "mjpeg": {"age_sec": 0.247, "exists": true, "mtime": "2026-07-01T19:33:58.180446+09:00", "mtime_epoch": 1782902038.1804464, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183993}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:33:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 898.32, "name": "celery", "pid": 237, "rss_mb": 892.125, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.01, "pushed": 4.01, "read": 3.977}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.717, "count": 119, "max_ms": 13.549, "p95_ms": 10.951}, "gstreamer_push": {"avg_ms": 0.889, "count": 120, "max_ms": 1.841, "p95_ms": 0.911}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.574, "count": 119, "max_ms": 67.694, "p95_ms": 58.886}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.535, "count": 119, "max_ms": 95.294, "p95_ms": 90.774}, "yolo11_640_fire_11class": {"avg_ms": 51.904, "count": 119, "max_ms": 76.378, "p95_ms": 68.829}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.779, "count": 119, "max_ms": 94.215, "p95_ms": 89.469}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.595, "count": 120, "max_ms": 314.614, "p95_ms": 273.628}, "read_interval": {"avg_ms": 251.191, "count": 119, "max_ms": 324.856, "p95_ms": 286.001}, "triton_wait": {"avg_ms": 67.871, "count": 119, "max_ms": 95.31, "p95_ms": 90.787}, "visualize": {"avg_ms": 11.415, "count": 119, "max_ms": 15.514, "p95_ms": 12.687}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 580}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4148.441, "limit_mb": 8796093022207.996}, "detail": {"pid": 19325, "restart_count": 146, "start_count": 147}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 123, "h264_decode_error_count": 128, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:30:10+09:00", "last_error_summary": "[h264 @ 0x556f40cfe3c0] error while decoding MB 22 25, bytestream -9", "last_exit_at": "2026-07-01T19:34:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:34:00+09:00", "pid": 19325, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.504, "name": "ffmpeg", "pid": 19325, "rss_mb": 136.504, "state": "S (sleeping)", "threads": 106}, "restart_count": 146, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.254, "sample_count": 1129, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 69658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:33:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.453, "last_frame_pushed_at": "2026-07-01T19:33:58+09:00", "last_frame_read_age_sec": 4.699, "last_frame_read_at": "2026-07-01T19:33:58+09:00", "mjpeg": {"age_sec": 4.438, "exists": true, "mtime": "2026-07-01T19:33:58.436451+09:00", "mtime_epoch": 1782902038.4364507, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184223}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:34:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 898.32, "name": "celery", "pid": 237, "rss_mb": 892.125, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.407, "pushed": 3.407, "read": 3.374}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.85, "count": 101, "max_ms": 13.549, "p95_ms": 10.951}, "gstreamer_push": {"avg_ms": 0.894, "count": 102, "max_ms": 1.841, "p95_ms": 0.913}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.524, "count": 101, "max_ms": 67.694, "p95_ms": 58.066}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.394, "count": 101, "max_ms": 95.294, "p95_ms": 90.2}, "yolo11_640_fire_11class": {"avg_ms": 51.745, "count": 101, "max_ms": 76.378, "p95_ms": 67.091}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.757, "count": 101, "max_ms": 94.215, "p95_ms": 89.149}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.991, "count": 102, "max_ms": 314.614, "p95_ms": 273.628}, "read_interval": {"avg_ms": 252.632, "count": 101, "max_ms": 324.856, "p95_ms": 282.843}, "triton_wait": {"avg_ms": 67.751, "count": 101, "max_ms": 95.31, "p95_ms": 90.222}, "visualize": {"avg_ms": 11.537, "count": 101, "max_ms": 15.514, "p95_ms": 12.687}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4050.086, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fd3911e7c0] error while decoding MB 24 34, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 124, "h264_decode_error_count": 129, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:35:50+09:00", "last_error_summary": "[h264 @ 0x55fd3911e7c0] error while decoding MB 24 34, bytestream -7", "last_exit_at": "2026-07-01T19:34:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.53, "last_started_at": "2026-07-01T19:34:00+09:00", "pid": 19325, "process": {"exists": false, "pid": 19325}, "restart_count": 146, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.109, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70074, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:35:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T19:35:50+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T19:35:50+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T19:35:50.362266+09:00", "mtime_epoch": 1782902150.3622658, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177593}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:35:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 898.32, "name": "celery", "pid": 237, "rss_mb": 898.062, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.921, "pushed": 3.921, "read": 3.921}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.75, "count": 116, "max_ms": 12.087, "p95_ms": 10.542}, "gstreamer_push": {"avg_ms": 0.928, "count": 117, "max_ms": 2.949, "p95_ms": 1.801}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.201, "count": 116, "max_ms": 70.485, "p95_ms": 62.497}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.363, "count": 116, "max_ms": 92.475, "p95_ms": 90.531}, "yolo11_640_fire_11class": {"avg_ms": 53.302, "count": 116, "max_ms": 75.091, "p95_ms": 70.773}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.268, "count": 116, "max_ms": 92.905, "p95_ms": 89.679}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.057, "count": 117, "max_ms": 323.427, "p95_ms": 278.16}, "read_interval": {"avg_ms": 256.717, "count": 117, "max_ms": 332.557, "p95_ms": 287.197}, "triton_wait": {"avg_ms": 69.822, "count": 116, "max_ms": 92.916, "p95_ms": 91.32}, "visualize": {"avg_ms": 11.435, "count": 116, "max_ms": 12.86, "p95_ms": 12.299}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.841, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4049.75, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:35:50+09:00", "last_error_summary": "[h264 @ 0x55fd3911e7c0] error while decoding MB 24 34, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 124, "h264_decode_error_count": 129, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:35:50+09:00", "last_error_summary": "[h264 @ 0x55fd3911e7c0] error while decoding MB 24 34, bytestream -7", "last_exit_at": "2026-07-01T19:35:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.734, "last_started_at": "2026-07-01T19:34:00+09:00", "pid": 19325, "process": {"exists": false, "pid": 19325}, "restart_count": 146, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.109, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70075, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:35:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:35:50+09:00", "last_frame_read_age_sec": 0.255, "last_frame_read_at": "2026-07-01T19:35:50+09:00", "mjpeg": {"age_sec": 0.246, "exists": true, "mtime": "2026-07-01T19:35:50.362266+09:00", "mtime_epoch": 1782902150.3622658, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177593}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:35:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 898.32, "name": "celery", "pid": 237, "rss_mb": 898.062, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.928, "pushed": 3.928, "read": 3.894}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.756, "count": 117, "max_ms": 12.087, "p95_ms": 10.542}, "gstreamer_push": {"avg_ms": 0.928, "count": 117, "max_ms": 2.949, "p95_ms": 1.801}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.187, "count": 117, "max_ms": 70.485, "p95_ms": 62.497}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.37, "count": 117, "max_ms": 92.475, "p95_ms": 90.531}, "yolo11_640_fire_11class": {"avg_ms": 53.307, "count": 117, "max_ms": 75.091, "p95_ms": 70.773}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.274, "count": 117, "max_ms": 92.905, "p95_ms": 89.679}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.101, "count": 117, "max_ms": 323.427, "p95_ms": 278.16}, "read_interval": {"avg_ms": 256.734, "count": 116, "max_ms": 332.557, "p95_ms": 287.197}, "triton_wait": {"avg_ms": 69.825, "count": 117, "max_ms": 92.916, "p95_ms": 91.32}, "visualize": {"avg_ms": 11.441, "count": 117, "max_ms": 12.86, "p95_ms": 12.299}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.788, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4049.867, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 124, "h264_decode_error_count": 129, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:35:50+09:00", "last_error_summary": "[h264 @ 0x55fd3911e7c0] error while decoding MB 24 34, bytestream -7", "last_exit_at": "2026-07-01T19:35:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.738, "last_started_at": "2026-07-01T19:34:00+09:00", "pid": 19325, "process": {"exists": false, "pid": 19325}, "restart_count": 146, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 147}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.109, "sample_count": 1142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70075, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:35:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:35:50+09:00", "last_frame_read_age_sec": 0.258, "last_frame_read_at": "2026-07-01T19:35:50+09:00", "mjpeg": {"age_sec": 0.249, "exists": true, "mtime": "2026-07-01T19:35:50.362266+09:00", "mtime_epoch": 1782902150.3622658, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177593}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:35:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 898.32, "name": "celery", "pid": 237, "rss_mb": 898.062, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.927, "pushed": 3.927, "read": 3.894}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.756, "count": 117, "max_ms": 12.087, "p95_ms": 10.542}, "gstreamer_push": {"avg_ms": 0.928, "count": 117, "max_ms": 2.949, "p95_ms": 1.801}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.187, "count": 117, "max_ms": 70.485, "p95_ms": 62.497}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.37, "count": 117, "max_ms": 92.475, "p95_ms": 90.531}, "yolo11_640_fire_11class": {"avg_ms": 53.307, "count": 117, "max_ms": 75.091, "p95_ms": 70.773}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.274, "count": 117, "max_ms": 92.905, "p95_ms": 89.679}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.101, "count": 117, "max_ms": 323.427, "p95_ms": 278.16}, "read_interval": {"avg_ms": 256.734, "count": 116, "max_ms": 332.557, "p95_ms": 287.197}, "triton_wait": {"avg_ms": 69.825, "count": 117, "max_ms": 92.916, "p95_ms": 91.32}, "visualize": {"avg_ms": 11.441, "count": 117, "max_ms": 12.86, "p95_ms": 12.299}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.792, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 583}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4157.234, "limit_mb": 8796093022207.996}, "detail": {"pid": 19432, "restart_count": 147, "start_count": 148}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 124, "h264_decode_error_count": 129, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:35:50+09:00", "last_error_summary": "[h264 @ 0x55fd3911e7c0] error while decoding MB 24 34, bytestream -7", "last_exit_at": "2026-07-01T19:35:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:35:53+09:00", "pid": 19432, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.406, "name": "ffmpeg", "pid": 19432, "rss_mb": 136.406, "state": "S (sleeping)", "threads": 106}, "restart_count": 147, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 278.109, "sample_count": 1125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70075, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:35:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.465, "last_frame_pushed_at": "2026-07-01T19:35:50+09:00", "last_frame_read_age_sec": 4.717, "last_frame_read_at": "2026-07-01T19:35:50+09:00", "mjpeg": {"age_sec": 4.448, "exists": true, "mtime": "2026-07-01T19:35:50.622270+09:00", "mtime_epoch": 1782902150.62227, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177896}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:35:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 898.32, "name": "celery", "pid": 237, "rss_mb": 898.062, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.357, "pushed": 3.357, "read": 3.324}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.757, "count": 100, "max_ms": 12.087, "p95_ms": 10.509}, "gstreamer_push": {"avg_ms": 0.918, "count": 100, "max_ms": 1.87, "p95_ms": 1.761}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.009, "count": 100, "max_ms": 70.485, "p95_ms": 62.497}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.03, "count": 100, "max_ms": 92.475, "p95_ms": 90.531}, "yolo11_640_fire_11class": {"avg_ms": 54.2, "count": 100, "max_ms": 75.091, "p95_ms": 70.773}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.028, "count": 100, "max_ms": 92.905, "p95_ms": 89.679}}, "truncated_count": 0}, "processing_total": {"avg_ms": 246.383, "count": 100, "max_ms": 317.383, "p95_ms": 277.782}, "read_interval": {"avg_ms": 255.974, "count": 99, "max_ms": 327.467, "p95_ms": 287.197}, "triton_wait": {"avg_ms": 70.415, "count": 100, "max_ms": 92.916, "p95_ms": 91.32}, "visualize": {"avg_ms": 11.396, "count": 100, "max_ms": 12.86, "p95_ms": 12.158}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.785, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4047.887, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56547fcc0c80] error while decoding MB 27 17, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 125, "h264_decode_error_count": 130, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:37:42+09:00", "last_error_summary": "[h264 @ 0x56547fcc0c80] error while decoding MB 27 17, bytestream -7", "last_exit_at": "2026-07-01T19:35:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.01, "last_started_at": "2026-07-01T19:35:53+09:00", "pid": 19432, "process": {"exists": false, "pid": 19432}, "restart_count": 147, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.591, "sample_count": 1159, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70511, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:37:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T19:37:42+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T19:37:42+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T19:37:42.036058+09:00", "mtime_epoch": 1782902262.0360577, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174941}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:37:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 892.762, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.075, "pushed": 4.075, "read": 4.075}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.379, "count": 121, "max_ms": 16.218, "p95_ms": 11.125}, "gstreamer_push": {"avg_ms": 0.923, "count": 122, "max_ms": 2.702, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.989, "count": 121, "max_ms": 70.897, "p95_ms": 62.355}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.966, "count": 121, "max_ms": 98.054, "p95_ms": 90.407}, "yolo11_640_fire_11class": {"avg_ms": 54.017, "count": 121, "max_ms": 77.954, "p95_ms": 70.634}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.824, "count": 121, "max_ms": 96.84, "p95_ms": 89.343}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.547, "count": 122, "max_ms": 316.733, "p95_ms": 275.224}, "read_interval": {"avg_ms": 246.841, "count": 122, "max_ms": 326.98, "p95_ms": 284.37}, "triton_wait": {"avg_ms": 69.479, "count": 121, "max_ms": 98.067, "p95_ms": 90.581}, "visualize": {"avg_ms": 11.161, "count": 121, "max_ms": 15.658, "p95_ms": 12.442}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4047.168, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:37:42+09:00", "last_error_summary": "[h264 @ 0x56547fcc0c80] error while decoding MB 27 17, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 125, "h264_decode_error_count": 130, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:37:42+09:00", "last_error_summary": "[h264 @ 0x56547fcc0c80] error while decoding MB 27 17, bytestream -7", "last_exit_at": "2026-07-01T19:37:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.191, "last_started_at": "2026-07-01T19:35:53+09:00", "pid": 19432, "process": {"exists": false, "pid": 19432}, "restart_count": 147, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.591, "sample_count": 1160, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70512, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:37:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:37:42+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T19:37:42+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T19:37:42.036058+09:00", "mtime_epoch": 1782902262.0360577, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174941}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:37:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 892.762, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.084, "pushed": 4.084, "read": 4.05}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.38, "count": 121, "max_ms": 16.218, "p95_ms": 11.125}, "gstreamer_push": {"avg_ms": 0.923, "count": 122, "max_ms": 2.702, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.876, "count": 121, "max_ms": 70.897, "p95_ms": 62.355}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.936, "count": 121, "max_ms": 98.054, "p95_ms": 90.407}, "yolo11_640_fire_11class": {"avg_ms": 53.999, "count": 121, "max_ms": 77.954, "p95_ms": 70.634}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.797, "count": 121, "max_ms": 96.84, "p95_ms": 89.343}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.538, "count": 122, "max_ms": 316.733, "p95_ms": 275.224}, "read_interval": {"avg_ms": 246.906, "count": 121, "max_ms": 326.98, "p95_ms": 284.37}, "triton_wait": {"avg_ms": 69.448, "count": 121, "max_ms": 98.067, "p95_ms": 90.581}, "visualize": {"avg_ms": 11.159, "count": 121, "max_ms": 15.658, "p95_ms": 12.442}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4047.16, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 125, "h264_decode_error_count": 130, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:37:42+09:00", "last_error_summary": "[h264 @ 0x56547fcc0c80] error while decoding MB 27 17, bytestream -7", "last_exit_at": "2026-07-01T19:37:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.194, "last_started_at": "2026-07-01T19:35:53+09:00", "pid": 19432, "process": {"exists": false, "pid": 19432}, "restart_count": 147, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 148}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.591, "sample_count": 1160, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70512, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:37:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:37:42+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T19:37:42+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T19:37:42.036058+09:00", "mtime_epoch": 1782902262.0360577, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174941}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:37:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 892.762, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.083, "pushed": 4.083, "read": 4.05}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.38, "count": 121, "max_ms": 16.218, "p95_ms": 11.125}, "gstreamer_push": {"avg_ms": 0.923, "count": 122, "max_ms": 2.702, "p95_ms": 0.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.876, "count": 121, "max_ms": 70.897, "p95_ms": 62.355}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.936, "count": 121, "max_ms": 98.054, "p95_ms": 90.407}, "yolo11_640_fire_11class": {"avg_ms": 53.999, "count": 121, "max_ms": 77.954, "p95_ms": 70.634}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.797, "count": 121, "max_ms": 96.84, "p95_ms": 89.343}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.538, "count": 122, "max_ms": 316.733, "p95_ms": 275.224}, "read_interval": {"avg_ms": 246.906, "count": 121, "max_ms": 326.98, "p95_ms": 284.37}, "triton_wait": {"avg_ms": 69.448, "count": 121, "max_ms": 98.067, "p95_ms": 90.581}, "visualize": {"avg_ms": 11.159, "count": 121, "max_ms": 15.658, "p95_ms": 12.442}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.877, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 587}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4147.066, "limit_mb": 8796093022207.996}, "detail": {"pid": 19539, "restart_count": 148, "start_count": 149}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 125, "h264_decode_error_count": 130, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:37:42+09:00", "last_error_summary": "[h264 @ 0x56547fcc0c80] error while decoding MB 27 17, bytestream -7", "last_exit_at": "2026-07-01T19:37:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:37:45+09:00", "pid": 19539, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.32, "name": "ffmpeg", "pid": 19539, "rss_mb": 139.32, "state": "S (sleeping)", "threads": 106}, "restart_count": 148, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 149}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.782, "sample_count": 1141, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70512, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:37:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.841, "last_frame_pushed_at": "2026-07-01T19:37:42+09:00", "last_frame_read_age_sec": 5.07, "last_frame_read_at": "2026-07-01T19:37:42+09:00", "mjpeg": {"age_sec": 4.826, "exists": true, "mtime": "2026-07-01T19:37:42.272062+09:00", "mtime_epoch": 1782902262.2720616, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175259}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:37:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 892.762, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.435, "pushed": 3.435, "read": 3.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.436, "count": 102, "max_ms": 16.218, "p95_ms": 11.179}, "gstreamer_push": {"avg_ms": 0.906, "count": 103, "max_ms": 1.986, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.853, "count": 102, "max_ms": 70.059, "p95_ms": 62.212}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.674, "count": 102, "max_ms": 98.054, "p95_ms": 86.013}, "yolo11_640_fire_11class": {"avg_ms": 53.92, "count": 102, "max_ms": 77.954, "p95_ms": 69.51}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.607, "count": 102, "max_ms": 96.84, "p95_ms": 89.343}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.186, "count": 103, "max_ms": 302.613, "p95_ms": 272.575}, "read_interval": {"avg_ms": 246.513, "count": 102, "max_ms": 313.215, "p95_ms": 283.703}, "triton_wait": {"avg_ms": 69.196, "count": 102, "max_ms": 98.067, "p95_ms": 90.423}, "visualize": {"avg_ms": 11.242, "count": 102, "max_ms": 15.658, "p95_ms": 13.252}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 591}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4042.824, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:37:42+09:00", "last_error_summary": "[h264 @ 0x56547fcc0c80] error while decoding MB 27 17, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 125, "h264_decode_error_count": 130, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:37:42+09:00", "last_error_summary": "[h264 @ 0x56547fcc0c80] error while decoding MB 27 17, bytestream -7", "last_exit_at": "2026-07-01T19:39:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.691, "last_started_at": "2026-07-01T19:37:45+09:00", "pid": 19539, "process": {"exists": false, "pid": 19539}, "restart_count": 148, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 149}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.355, "sample_count": 1151, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70935, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:39:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:39:35+09:00", "last_frame_read_age_sec": 0.223, "last_frame_read_at": "2026-07-01T19:39:35+09:00", "mjpeg": {"age_sec": 0.215, "exists": true, "mtime": "2026-07-01T19:39:35.573864+09:00", "mtime_epoch": 1782902375.5738637, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174136}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:39:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.098, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.018, "pushed": 4.018, "read": 3.984}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.702, "count": 119, "max_ms": 16.238, "p95_ms": 11.761}, "gstreamer_push": {"avg_ms": 0.912, "count": 120, "max_ms": 2.005, "p95_ms": 1.02}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.135, "count": 119, "max_ms": 68.967, "p95_ms": 61.082}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.896, "count": 119, "max_ms": 89.051, "p95_ms": 84.67}, "yolo11_640_fire_11class": {"avg_ms": 55.902, "count": 119, "max_ms": 74.32, "p95_ms": 69.61}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.975, "count": 119, "max_ms": 91.119, "p95_ms": 85.951}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.763, "count": 120, "max_ms": 319.544, "p95_ms": 270.46}, "read_interval": {"avg_ms": 251.52, "count": 119, "max_ms": 329.481, "p95_ms": 280.751}, "triton_wait": {"avg_ms": 68.342, "count": 119, "max_ms": 91.13, "p95_ms": 87.097}, "visualize": {"avg_ms": 11.399, "count": 119, "max_ms": 16.497, "p95_ms": 13.041}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 591}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4042.816, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 125, "h264_decode_error_count": 130, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:37:42+09:00", "last_error_summary": "[h264 @ 0x56547fcc0c80] error while decoding MB 27 17, bytestream -7", "last_exit_at": "2026-07-01T19:39:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.695, "last_started_at": "2026-07-01T19:37:45+09:00", "pid": 19539, "process": {"exists": false, "pid": 19539}, "restart_count": 148, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 149}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.355, "sample_count": 1151, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70935, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:39:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:39:35+09:00", "last_frame_read_age_sec": 0.226, "last_frame_read_at": "2026-07-01T19:39:35+09:00", "mjpeg": {"age_sec": 0.218, "exists": true, "mtime": "2026-07-01T19:39:35.573864+09:00", "mtime_epoch": 1782902375.5738637, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174136}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:39:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.098, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.017, "pushed": 4.017, "read": 3.984}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.702, "count": 119, "max_ms": 16.238, "p95_ms": 11.761}, "gstreamer_push": {"avg_ms": 0.912, "count": 120, "max_ms": 2.005, "p95_ms": 1.02}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.135, "count": 119, "max_ms": 68.967, "p95_ms": 61.082}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.896, "count": 119, "max_ms": 89.051, "p95_ms": 84.67}, "yolo11_640_fire_11class": {"avg_ms": 55.902, "count": 119, "max_ms": 74.32, "p95_ms": 69.61}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.975, "count": 119, "max_ms": 91.119, "p95_ms": 85.951}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.763, "count": 120, "max_ms": 319.544, "p95_ms": 270.46}, "read_interval": {"avg_ms": 251.52, "count": 119, "max_ms": 329.481, "p95_ms": 280.751}, "triton_wait": {"avg_ms": 68.342, "count": 119, "max_ms": 91.13, "p95_ms": 87.097}, "visualize": {"avg_ms": 11.399, "count": 119, "max_ms": 16.497, "p95_ms": 13.041}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 591}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4153.586, "limit_mb": 8796093022207.996}, "detail": {"pid": 19646, "restart_count": 149, "start_count": 150}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 125, "h264_decode_error_count": 130, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:37:42+09:00", "last_error_summary": "[h264 @ 0x56547fcc0c80] error while decoding MB 27 17, bytestream -7", "last_exit_at": "2026-07-01T19:39:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:39:38+09:00", "pid": 19646, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.23, "name": "ffmpeg", "pid": 19646, "rss_mb": 139.23, "state": "S (sleeping)", "threads": 106}, "restart_count": 149, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.307, "sample_count": 1134, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70935, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:39:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.501, "last_frame_pushed_at": "2026-07-01T19:39:35+09:00", "last_frame_read_age_sec": 4.721, "last_frame_read_at": "2026-07-01T19:39:35+09:00", "mjpeg": {"age_sec": 4.485, "exists": true, "mtime": "2026-07-01T19:39:35.801867+09:00", "mtime_epoch": 1782902375.8018675, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174315}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:39:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.098, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.392, "pushed": 3.392, "read": 3.359}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.845, "count": 101, "max_ms": 16.238, "p95_ms": 11.761}, "gstreamer_push": {"avg_ms": 0.909, "count": 101, "max_ms": 1.922, "p95_ms": 1.02}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.58, "count": 101, "max_ms": 68.967, "p95_ms": 60.53}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.641, "count": 101, "max_ms": 89.051, "p95_ms": 84.505}, "yolo11_640_fire_11class": {"avg_ms": 55.512, "count": 101, "max_ms": 74.32, "p95_ms": 69.61}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.627, "count": 101, "max_ms": 91.119, "p95_ms": 85.951}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.094, "count": 101, "max_ms": 319.544, "p95_ms": 270.46}, "read_interval": {"avg_ms": 252.884, "count": 100, "max_ms": 329.481, "p95_ms": 279.653}, "triton_wait": {"avg_ms": 68.095, "count": 101, "max_ms": 91.13, "p95_ms": 87.097}, "visualize": {"avg_ms": 11.488, "count": 101, "max_ms": 16.497, "p95_ms": 13.041}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.774, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 594}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4047.035, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560412633ec0] error while decoding MB 5 19, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 126, "h264_decode_error_count": 131, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:41:27+09:00", "last_error_summary": "[h264 @ 0x560412633ec0] error while decoding MB 5 19, bytestream -5", "last_exit_at": "2026-07-01T19:39:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.011, "last_started_at": "2026-07-01T19:39:38+09:00", "pid": 19646, "process": {"exists": false, "pid": 19646}, "restart_count": 149, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.296, "sample_count": 1149, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71356, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:41:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T19:41:27+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T19:41:27+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T19:41:27.255628+09:00", "mtime_epoch": 1782902487.2556279, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:41:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.254, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.116, "pushed": 4.116, "read": 4.116}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.292, "count": 122, "max_ms": 11.317, "p95_ms": 10.636}, "gstreamer_push": {"avg_ms": 0.962, "count": 123, "max_ms": 2.901, "p95_ms": 1.862}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.452, "count": 122, "max_ms": 71.488, "p95_ms": 60.49}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.792, "count": 122, "max_ms": 91.94, "p95_ms": 88.708}, "yolo11_640_fire_11class": {"avg_ms": 53.33, "count": 122, "max_ms": 74.048, "p95_ms": 68.037}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.876, "count": 122, "max_ms": 91.95, "p95_ms": 87.226}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.719, "count": 123, "max_ms": 335.874, "p95_ms": 257.13}, "read_interval": {"avg_ms": 244.381, "count": 123, "max_ms": 346.017, "p95_ms": 266.237}, "triton_wait": {"avg_ms": 67.63, "count": 122, "max_ms": 91.962, "p95_ms": 88.767}, "visualize": {"avg_ms": 11.031, "count": 122, "max_ms": 14.412, "p95_ms": 12.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 594}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4046.23, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:41:27+09:00", "last_error_summary": "[h264 @ 0x560412633ec0] error while decoding MB 5 19, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 126, "h264_decode_error_count": 131, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:41:27+09:00", "last_error_summary": "[h264 @ 0x560412633ec0] error while decoding MB 5 19, bytestream -5", "last_exit_at": "2026-07-01T19:41:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.194, "last_started_at": "2026-07-01T19:39:38+09:00", "pid": 19646, "process": {"exists": false, "pid": 19646}, "restart_count": 149, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.296, "sample_count": 1150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71357, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:41:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:41:27+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T19:41:27+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T19:41:27.255628+09:00", "mtime_epoch": 1782902487.2556279, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:41:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.254, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.127, "pushed": 4.127, "read": 4.094}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.29, "count": 123, "max_ms": 11.317, "p95_ms": 10.636}, "gstreamer_push": {"avg_ms": 0.963, "count": 123, "max_ms": 2.901, "p95_ms": 1.862}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.472, "count": 123, "max_ms": 71.488, "p95_ms": 60.49}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.811, "count": 123, "max_ms": 91.94, "p95_ms": 88.708}, "yolo11_640_fire_11class": {"avg_ms": 53.362, "count": 123, "max_ms": 74.048, "p95_ms": 68.037}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.894, "count": 123, "max_ms": 91.95, "p95_ms": 87.226}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.749, "count": 123, "max_ms": 335.874, "p95_ms": 257.13}, "read_interval": {"avg_ms": 244.447, "count": 122, "max_ms": 346.017, "p95_ms": 266.237}, "triton_wait": {"avg_ms": 67.642, "count": 123, "max_ms": 91.962, "p95_ms": 88.767}, "visualize": {"avg_ms": 11.032, "count": 123, "max_ms": 14.412, "p95_ms": 12.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.802, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 594}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4046.223, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 126, "h264_decode_error_count": 131, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:41:27+09:00", "last_error_summary": "[h264 @ 0x560412633ec0] error while decoding MB 5 19, bytestream -5", "last_exit_at": "2026-07-01T19:41:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.197, "last_started_at": "2026-07-01T19:39:38+09:00", "pid": 19646, "process": {"exists": false, "pid": 19646}, "restart_count": 149, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 150}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.296, "sample_count": 1150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71357, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:41:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:41:27+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T19:41:27+09:00", "mjpeg": {"age_sec": 0.227, "exists": true, "mtime": "2026-07-01T19:41:27.255628+09:00", "mtime_epoch": 1782902487.2556279, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172456}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:41:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.254, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.127, "pushed": 4.127, "read": 4.093}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.29, "count": 123, "max_ms": 11.317, "p95_ms": 10.636}, "gstreamer_push": {"avg_ms": 0.963, "count": 123, "max_ms": 2.901, "p95_ms": 1.862}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.472, "count": 123, "max_ms": 71.488, "p95_ms": 60.49}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.811, "count": 123, "max_ms": 91.94, "p95_ms": 88.708}, "yolo11_640_fire_11class": {"avg_ms": 53.362, "count": 123, "max_ms": 74.048, "p95_ms": 68.037}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.894, "count": 123, "max_ms": 91.95, "p95_ms": 87.226}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.749, "count": 123, "max_ms": 335.874, "p95_ms": 257.13}, "read_interval": {"avg_ms": 244.447, "count": 122, "max_ms": 346.017, "p95_ms": 266.237}, "triton_wait": {"avg_ms": 67.642, "count": 123, "max_ms": 91.962, "p95_ms": 88.767}, "visualize": {"avg_ms": 11.032, "count": 123, "max_ms": 14.412, "p95_ms": 12.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.805, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 594}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4154.109, "limit_mb": 8796093022207.996}, "detail": {"pid": 19753, "restart_count": 150, "start_count": 151}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 126, "h264_decode_error_count": 131, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:41:27+09:00", "last_error_summary": "[h264 @ 0x560412633ec0] error while decoding MB 5 19, bytestream -5", "last_exit_at": "2026-07-01T19:41:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:41:30+09:00", "pid": 19753, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.23, "name": "ffmpeg", "pid": 19753, "rss_mb": 139.23, "state": "S (sleeping)", "threads": 106}, "restart_count": 150, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.296, "sample_count": 1129, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71357, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:41:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.944, "last_frame_pushed_at": "2026-07-01T19:41:27+09:00", "last_frame_read_age_sec": 5.176, "last_frame_read_at": "2026-07-01T19:41:27+09:00", "mjpeg": {"age_sec": 4.925, "exists": true, "mtime": "2026-07-01T19:41:27.495632+09:00", "mtime_epoch": 1782902487.4956317, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165584}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:41:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.254, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.456, "pushed": 3.456, "read": 3.422}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.157, "count": 103, "max_ms": 10.94, "p95_ms": 10.037}, "gstreamer_push": {"avg_ms": 0.972, "count": 103, "max_ms": 2.901, "p95_ms": 1.862}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.042, "count": 103, "max_ms": 65.386, "p95_ms": 60.49}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.015, "count": 103, "max_ms": 91.94, "p95_ms": 88.708}, "yolo11_640_fire_11class": {"avg_ms": 53.711, "count": 103, "max_ms": 74.048, "p95_ms": 68.037}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.17, "count": 103, "max_ms": 90.588, "p95_ms": 87.226}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.162, "count": 103, "max_ms": 312.014, "p95_ms": 253.84}, "read_interval": {"avg_ms": 243.861, "count": 102, "max_ms": 324.593, "p95_ms": 264.171}, "triton_wait": {"avg_ms": 67.902, "count": 103, "max_ms": 91.951, "p95_ms": 88.767}, "visualize": {"avg_ms": 10.965, "count": 103, "max_ms": 12.568, "p95_ms": 11.917}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.803, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 598}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4047.91, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56401a051f80] error while decoding MB 55 31, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 127, "h264_decode_error_count": 132, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:43:19+09:00", "last_error_summary": "[h264 @ 0x56401a051f80] error while decoding MB 55 31, bytestream -9", "last_exit_at": "2026-07-01T19:41:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.093, "last_started_at": "2026-07-01T19:41:30+09:00", "pid": 19753, "process": {"exists": false, "pid": 19753}, "restart_count": 150, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.733, "sample_count": 1144, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71781, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:43:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T19:43:19+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T19:43:19+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T19:43:19.469390+09:00", "mtime_epoch": 1782902599.4693904, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170153}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:43:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.989, "pushed": 3.989, "read": 3.989}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.607, "count": 119, "max_ms": 12.852, "p95_ms": 11.86}, "gstreamer_push": {"avg_ms": 0.952, "count": 119, "max_ms": 2.877, "p95_ms": 1.841}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.422, "count": 119, "max_ms": 71.219, "p95_ms": 65.114}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.771, "count": 119, "max_ms": 91.875, "p95_ms": 88.315}, "yolo11_640_fire_11class": {"avg_ms": 53.763, "count": 119, "max_ms": 71.887, "p95_ms": 69.512}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.219, "count": 119, "max_ms": 91.175, "p95_ms": 88.413}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.673, "count": 119, "max_ms": 345.971, "p95_ms": 276.054}, "read_interval": {"avg_ms": 252.137, "count": 119, "max_ms": 356.982, "p95_ms": 287.148}, "triton_wait": {"avg_ms": 68.563, "count": 119, "max_ms": 91.887, "p95_ms": 89.982}, "visualize": {"avg_ms": 11.287, "count": 119, "max_ms": 14.618, "p95_ms": 13.29}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.836, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 598}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4047.465, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:43:19+09:00", "last_error_summary": "[h264 @ 0x56401a051f80] error while decoding MB 55 31, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 127, "h264_decode_error_count": 132, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:43:19+09:00", "last_error_summary": "[h264 @ 0x56401a051f80] error while decoding MB 55 31, bytestream -9", "last_exit_at": "2026-07-01T19:43:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.277, "last_started_at": "2026-07-01T19:41:30+09:00", "pid": 19753, "process": {"exists": false, "pid": 19753}, "restart_count": 150, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.733, "sample_count": 1144, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71782, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:43:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:43:19+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T19:43:19+09:00", "mjpeg": {"age_sec": 0.227, "exists": true, "mtime": "2026-07-01T19:43:19.469390+09:00", "mtime_epoch": 1782902599.4693904, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170153}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:43:19+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.551, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.996, "pushed": 3.996, "read": 3.963}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.604, "count": 119, "max_ms": 12.852, "p95_ms": 11.86}, "gstreamer_push": {"avg_ms": 0.952, "count": 119, "max_ms": 2.877, "p95_ms": 1.841}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.483, "count": 119, "max_ms": 71.219, "p95_ms": 65.114}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.855, "count": 119, "max_ms": 91.875, "p95_ms": 88.315}, "yolo11_640_fire_11class": {"avg_ms": 53.844, "count": 119, "max_ms": 71.887, "p95_ms": 69.512}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.303, "count": 119, "max_ms": 91.175, "p95_ms": 88.413}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.764, "count": 119, "max_ms": 345.971, "p95_ms": 276.054}, "read_interval": {"avg_ms": 252.339, "count": 118, "max_ms": 356.982, "p95_ms": 287.148}, "triton_wait": {"avg_ms": 68.647, "count": 119, "max_ms": 91.887, "p95_ms": 89.982}, "visualize": {"avg_ms": 11.285, "count": 119, "max_ms": 14.618, "p95_ms": 13.29}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.778, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 598}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4047.465, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 127, "h264_decode_error_count": 132, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:43:19+09:00", "last_error_summary": "[h264 @ 0x56401a051f80] error while decoding MB 55 31, bytestream -9", "last_exit_at": "2026-07-01T19:43:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.28, "last_started_at": "2026-07-01T19:41:30+09:00", "pid": 19753, "process": {"exists": false, "pid": 19753}, "restart_count": 150, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 151}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.733, "sample_count": 1144, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71782, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:43:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:43:19+09:00", "last_frame_read_age_sec": 0.237, "last_frame_read_at": "2026-07-01T19:43:19+09:00", "mjpeg": {"age_sec": 0.23, "exists": true, "mtime": "2026-07-01T19:43:19.469390+09:00", "mtime_epoch": 1782902599.4693904, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170153}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:43:19+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.551, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 118, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.996, "pushed": 3.996, "read": 3.962}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.604, "count": 119, "max_ms": 12.852, "p95_ms": 11.86}, "gstreamer_push": {"avg_ms": 0.952, "count": 119, "max_ms": 2.877, "p95_ms": 1.841}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.483, "count": 119, "max_ms": 71.219, "p95_ms": 65.114}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.855, "count": 119, "max_ms": 91.875, "p95_ms": 88.315}, "yolo11_640_fire_11class": {"avg_ms": 53.844, "count": 119, "max_ms": 71.887, "p95_ms": 69.512}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.303, "count": 119, "max_ms": 91.175, "p95_ms": 88.413}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.764, "count": 119, "max_ms": 345.971, "p95_ms": 276.054}, "read_interval": {"avg_ms": 252.339, "count": 118, "max_ms": 356.982, "p95_ms": 287.148}, "triton_wait": {"avg_ms": 68.647, "count": 119, "max_ms": 91.887, "p95_ms": 89.982}, "visualize": {"avg_ms": 11.285, "count": 119, "max_ms": 14.618, "p95_ms": 13.29}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.781, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 598}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4155.664, "limit_mb": 8796093022207.996}, "detail": {"pid": 19860, "restart_count": 151, "start_count": 152}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 127, "h264_decode_error_count": 132, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:43:19+09:00", "last_error_summary": "[h264 @ 0x56401a051f80] error while decoding MB 55 31, bytestream -9", "last_exit_at": "2026-07-01T19:43:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:43:21+09:00", "pid": 19860, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.262, "name": "ffmpeg", "pid": 19860, "rss_mb": 139.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 151, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.733, "sample_count": 1129, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71782, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:43:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.021, "last_frame_pushed_at": "2026-07-01T19:43:19+09:00", "last_frame_read_age_sec": 4.252, "last_frame_read_at": "2026-07-01T19:43:19+09:00", "mjpeg": {"age_sec": 4.005, "exists": true, "mtime": "2026-07-01T19:43:19.709394+09:00", "mtime_epoch": 1782902599.7093942, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170341}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:43:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.469, "pushed": 3.469, "read": 3.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.629, "count": 103, "max_ms": 12.852, "p95_ms": 11.86}, "gstreamer_push": {"avg_ms": 0.936, "count": 104, "max_ms": 2.877, "p95_ms": 1.107}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.564, "count": 103, "max_ms": 71.219, "p95_ms": 65.114}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.665, "count": 103, "max_ms": 91.875, "p95_ms": 87.845}, "yolo11_640_fire_11class": {"avg_ms": 53.677, "count": 103, "max_ms": 71.887, "p95_ms": 68.614}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.224, "count": 103, "max_ms": 91.175, "p95_ms": 88.413}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.928, "count": 104, "max_ms": 345.971, "p95_ms": 273.955}, "read_interval": {"avg_ms": 252.459, "count": 103, "max_ms": 356.982, "p95_ms": 282.994}, "triton_wait": {"avg_ms": 68.541, "count": 103, "max_ms": 91.887, "p95_ms": 89.502}, "visualize": {"avg_ms": 11.29, "count": 103, "max_ms": 14.216, "p95_ms": 12.895}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 601}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4046.305, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56422eaf8dc0] error while decoding MB 52 7, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 128, "h264_decode_error_count": 133, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:45:10+09:00", "last_error_summary": "[h264 @ 0x56422eaf8dc0] error while decoding MB 52 7, bytestream -7", "last_exit_at": "2026-07-01T19:43:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.105, "last_started_at": "2026-07-01T19:43:21+09:00", "pid": 19860, "process": {"exists": false, "pid": 19860}, "restart_count": 151, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.054, "sample_count": 1152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72206, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:45:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T19:45:10+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T19:45:10+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T19:45:10.775131+09:00", "mtime_epoch": 1782902710.7751307, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170638}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:45:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 890.352, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.991, "pushed": 3.991, "read": 3.991}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.702, "count": 119, "max_ms": 11.53, "p95_ms": 10.84}, "gstreamer_push": {"avg_ms": 0.907, "count": 119, "max_ms": 1.865, "p95_ms": 0.99}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.359, "count": 119, "max_ms": 67.508, "p95_ms": 58.945}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.929, "count": 119, "max_ms": 93.016, "p95_ms": 88.851}, "yolo11_640_fire_11class": {"avg_ms": 52.322, "count": 119, "max_ms": 77.522, "p95_ms": 68.412}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.514, "count": 119, "max_ms": 91.93, "p95_ms": 88.685}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.682, "count": 119, "max_ms": 349.902, "p95_ms": 278.822}, "read_interval": {"avg_ms": 252.249, "count": 119, "max_ms": 360.162, "p95_ms": 287.828}, "triton_wait": {"avg_ms": 67.441, "count": 119, "max_ms": 93.028, "p95_ms": 88.96}, "visualize": {"avg_ms": 11.504, "count": 119, "max_ms": 15.91, "p95_ms": 12.797}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.818, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 601}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4048.621, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:45:10+09:00", "last_error_summary": "[h264 @ 0x56422eaf8dc0] error while decoding MB 52 7, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 128, "h264_decode_error_count": 133, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:45:10+09:00", "last_error_summary": "[h264 @ 0x56422eaf8dc0] error while decoding MB 52 7, bytestream -7", "last_exit_at": "2026-07-01T19:45:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.29, "last_started_at": "2026-07-01T19:43:21+09:00", "pid": 19860, "process": {"exists": false, "pid": 19860}, "restart_count": 151, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.054, "sample_count": 1153, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72207, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:45:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T19:45:10+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-01T19:45:10+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T19:45:10.775131+09:00", "mtime_epoch": 1782902710.7751307, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170638}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:45:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.961, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.968, "pushed": 3.968, "read": 3.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.717, "count": 119, "max_ms": 12.147, "p95_ms": 10.973}, "gstreamer_push": {"avg_ms": 0.91, "count": 119, "max_ms": 1.865, "p95_ms": 1.128}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.214, "count": 119, "max_ms": 67.508, "p95_ms": 58.945}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.838, "count": 119, "max_ms": 93.016, "p95_ms": 88.851}, "yolo11_640_fire_11class": {"avg_ms": 52.235, "count": 119, "max_ms": 77.522, "p95_ms": 68.412}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.417, "count": 119, "max_ms": 91.93, "p95_ms": 88.685}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.505, "count": 119, "max_ms": 349.902, "p95_ms": 278.822}, "read_interval": {"avg_ms": 252.249, "count": 119, "max_ms": 360.162, "p95_ms": 287.828}, "triton_wait": {"avg_ms": 67.344, "count": 119, "max_ms": 93.028, "p95_ms": 88.96}, "visualize": {"avg_ms": 11.526, "count": 119, "max_ms": 15.91, "p95_ms": 12.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 601}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4048.738, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 128, "h264_decode_error_count": 133, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:45:10+09:00", "last_error_summary": "[h264 @ 0x56422eaf8dc0] error while decoding MB 52 7, bytestream -7", "last_exit_at": "2026-07-01T19:45:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.294, "last_started_at": "2026-07-01T19:43:21+09:00", "pid": 19860, "process": {"exists": false, "pid": 19860}, "restart_count": 151, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 152}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.054, "sample_count": 1153, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72207, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:45:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T19:45:10+09:00", "last_frame_read_age_sec": 0.24, "last_frame_read_at": "2026-07-01T19:45:10+09:00", "mjpeg": {"age_sec": 0.232, "exists": true, "mtime": "2026-07-01T19:45:10.775131+09:00", "mtime_epoch": 1782902710.7751307, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170638}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:45:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.961, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.967, "pushed": 3.967, "read": 3.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.717, "count": 119, "max_ms": 12.147, "p95_ms": 10.973}, "gstreamer_push": {"avg_ms": 0.91, "count": 119, "max_ms": 1.865, "p95_ms": 1.128}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.214, "count": 119, "max_ms": 67.508, "p95_ms": 58.945}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.838, "count": 119, "max_ms": 93.016, "p95_ms": 88.851}, "yolo11_640_fire_11class": {"avg_ms": 52.235, "count": 119, "max_ms": 77.522, "p95_ms": 68.412}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.417, "count": 119, "max_ms": 91.93, "p95_ms": 88.685}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.505, "count": 119, "max_ms": 349.902, "p95_ms": 278.822}, "read_interval": {"avg_ms": 252.249, "count": 119, "max_ms": 360.162, "p95_ms": 287.828}, "triton_wait": {"avg_ms": 67.344, "count": 119, "max_ms": 93.028, "p95_ms": 88.96}, "visualize": {"avg_ms": 11.526, "count": 119, "max_ms": 15.91, "p95_ms": 12.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 601}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4156.316, "limit_mb": 8796093022207.996}, "detail": {"pid": 19967, "restart_count": 152, "start_count": 153}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 128, "h264_decode_error_count": 133, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:45:10+09:00", "last_error_summary": "[h264 @ 0x56422eaf8dc0] error while decoding MB 52 7, bytestream -7", "last_exit_at": "2026-07-01T19:45:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:45:13+09:00", "pid": 19967, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.879, "name": "ffmpeg", "pid": 19967, "rss_mb": 138.879, "state": "S (sleeping)", "threads": 106}, "restart_count": 152, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 153}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.163, "sample_count": 1134, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72207, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:45:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.919, "last_frame_pushed_at": "2026-07-01T19:45:10+09:00", "last_frame_read_age_sec": 5.15, "last_frame_read_at": "2026-07-01T19:45:10+09:00", "mjpeg": {"age_sec": 4.903, "exists": true, "mtime": "2026-07-01T19:45:11.015134+09:00", "mtime_epoch": 1782902711.0151343, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 170800}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:45:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 893.961, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.345, "pushed": 3.345, "read": 3.312}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.609, "count": 99, "max_ms": 12.147, "p95_ms": 10.84}, "gstreamer_push": {"avg_ms": 0.917, "count": 100, "max_ms": 1.865, "p95_ms": 1.128}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.256, "count": 99, "max_ms": 67.508, "p95_ms": 59.84}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.403, "count": 99, "max_ms": 93.016, "p95_ms": 89.24}, "yolo11_640_fire_11class": {"avg_ms": 52.497, "count": 99, "max_ms": 77.522, "p95_ms": 68.458}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.849, "count": 99, "max_ms": 91.93, "p95_ms": 88.948}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.515, "count": 100, "max_ms": 349.902, "p95_ms": 278.822}, "read_interval": {"avg_ms": 252.225, "count": 99, "max_ms": 360.162, "p95_ms": 293.314}, "triton_wait": {"avg_ms": 67.84, "count": 99, "max_ms": 93.028, "p95_ms": 90.177}, "visualize": {"avg_ms": 11.505, "count": 99, "max_ms": 15.91, "p95_ms": 12.975}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4060.824, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f115f84f80] error while decoding MB 7 40, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 129, "h264_decode_error_count": 134, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:47:03+09:00", "last_error_summary": "[h264 @ 0x55f115f84f80] error while decoding MB 7 40, bytestream -15", "last_exit_at": "2026-07-01T19:45:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.504, "last_started_at": "2026-07-01T19:45:13+09:00", "pid": 19967, "process": {"exists": false, "pid": 19967}, "restart_count": 152, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 153}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.733, "sample_count": 1156, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72635, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:47:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.051, "last_frame_pushed_at": "2026-07-01T19:47:03+09:00", "last_frame_read_age_sec": 0.042, "last_frame_read_at": "2026-07-01T19:47:03+09:00", "mjpeg": {"age_sec": 0.032, "exists": true, "mtime": "2026-07-01T19:47:03.388885+09:00", "mtime_epoch": 1782902823.388885, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168677}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:47:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 898.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.002, "pushed": 4.002, "read": 4.002}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.539, "count": 119, "max_ms": 11.339, "p95_ms": 10.383}, "gstreamer_push": {"avg_ms": 0.93, "count": 120, "max_ms": 1.98, "p95_ms": 1.742}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.216, "count": 119, "max_ms": 72.433, "p95_ms": 57.172}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.865, "count": 119, "max_ms": 94.863, "p95_ms": 89.158}, "yolo11_640_fire_11class": {"avg_ms": 52.933, "count": 119, "max_ms": 74.085, "p95_ms": 67.925}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.22, "count": 119, "max_ms": 94.306, "p95_ms": 89.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.871, "count": 120, "max_ms": 328.216, "p95_ms": 278.863}, "read_interval": {"avg_ms": 251.441, "count": 120, "max_ms": 338.715, "p95_ms": 287.991}, "triton_wait": {"avg_ms": 67.272, "count": 119, "max_ms": 94.872, "p95_ms": 90.067}, "visualize": {"avg_ms": 11.386, "count": 119, "max_ms": 16.236, "p95_ms": 12.441}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4059.859, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:47:03+09:00", "last_error_summary": "[h264 @ 0x55f115f84f80] error while decoding MB 7 40, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 129, "h264_decode_error_count": 134, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:47:03+09:00", "last_error_summary": "[h264 @ 0x55f115f84f80] error while decoding MB 7 40, bytestream -15", "last_exit_at": "2026-07-01T19:47:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.697, "last_started_at": "2026-07-01T19:45:13+09:00", "pid": 19967, "process": {"exists": false, "pid": 19967}, "restart_count": 152, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 153}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.733, "sample_count": 1156, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72636, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:47:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:47:03+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T19:47:03+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-01T19:47:03.388885+09:00", "mtime_epoch": 1782902823.388885, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168677}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:47:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 898.629, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.008, "pushed": 4.008, "read": 3.975}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.538, "count": 119, "max_ms": 11.339, "p95_ms": 10.383}, "gstreamer_push": {"avg_ms": 0.93, "count": 120, "max_ms": 1.98, "p95_ms": 1.742}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.216, "count": 119, "max_ms": 72.433, "p95_ms": 57.172}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.864, "count": 119, "max_ms": 94.863, "p95_ms": 89.158}, "yolo11_640_fire_11class": {"avg_ms": 52.903, "count": 119, "max_ms": 74.085, "p95_ms": 67.925}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.196, "count": 119, "max_ms": 94.306, "p95_ms": 89.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.872, "count": 120, "max_ms": 328.216, "p95_ms": 278.863}, "read_interval": {"avg_ms": 251.543, "count": 119, "max_ms": 338.715, "p95_ms": 291.404}, "triton_wait": {"avg_ms": 67.258, "count": 119, "max_ms": 94.872, "p95_ms": 90.067}, "visualize": {"avg_ms": 11.375, "count": 119, "max_ms": 16.236, "p95_ms": 12.419}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4059.852, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 129, "h264_decode_error_count": 134, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:47:03+09:00", "last_error_summary": "[h264 @ 0x55f115f84f80] error while decoding MB 7 40, bytestream -15", "last_exit_at": "2026-07-01T19:47:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.7, "last_started_at": "2026-07-01T19:45:13+09:00", "pid": 19967, "process": {"exists": false, "pid": 19967}, "restart_count": 152, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 153}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.733, "sample_count": 1156, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72636, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:47:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:47:03+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-01T19:47:03+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T19:47:03.388885+09:00", "mtime_epoch": 1782902823.388885, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168677}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:47:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 898.629, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.008, "pushed": 4.008, "read": 3.975}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.538, "count": 119, "max_ms": 11.339, "p95_ms": 10.383}, "gstreamer_push": {"avg_ms": 0.93, "count": 120, "max_ms": 1.98, "p95_ms": 1.742}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.216, "count": 119, "max_ms": 72.433, "p95_ms": 57.172}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.864, "count": 119, "max_ms": 94.863, "p95_ms": 89.158}, "yolo11_640_fire_11class": {"avg_ms": 52.903, "count": 119, "max_ms": 74.085, "p95_ms": 67.925}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.196, "count": 119, "max_ms": 94.306, "p95_ms": 89.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 241.872, "count": 120, "max_ms": 328.216, "p95_ms": 278.863}, "read_interval": {"avg_ms": 251.543, "count": 119, "max_ms": 338.715, "p95_ms": 291.404}, "triton_wait": {"avg_ms": 67.258, "count": 119, "max_ms": 94.872, "p95_ms": 90.067}, "visualize": {"avg_ms": 11.375, "count": 119, "max_ms": 16.236, "p95_ms": 12.419}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 605}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4165.02, "limit_mb": 8796093022207.996}, "detail": {"pid": 20074, "restart_count": 153, "start_count": 154}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 129, "h264_decode_error_count": 134, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:47:03+09:00", "last_error_summary": "[h264 @ 0x55f115f84f80] error while decoding MB 7 40, bytestream -15", "last_exit_at": "2026-07-01T19:47:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:47:06+09:00", "pid": 20074, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.738, "name": "ffmpeg", "pid": 20074, "rss_mb": 139.738, "state": "S (sleeping)", "threads": 106}, "restart_count": 153, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 154}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.231, "sample_count": 1137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72636, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:47:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.968, "last_frame_pushed_at": "2026-07-01T19:47:03+09:00", "last_frame_read_age_sec": 5.2, "last_frame_read_at": "2026-07-01T19:47:03+09:00", "mjpeg": {"age_sec": 4.95, "exists": true, "mtime": "2026-07-01T19:47:03.628889+09:00", "mtime_epoch": 1782902823.6288886, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168749}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:47:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 901.105, "name": "celery", "pid": 237, "rss_mb": 898.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.335, "pushed": 3.335, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.649, "count": 99, "max_ms": 11.339, "p95_ms": 10.458}, "gstreamer_push": {"avg_ms": 0.944, "count": 100, "max_ms": 1.98, "p95_ms": 1.763}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.073, "count": 99, "max_ms": 72.433, "p95_ms": 57.172}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.354, "count": 99, "max_ms": 94.863, "p95_ms": 90.035}, "yolo11_640_fire_11class": {"avg_ms": 52.33, "count": 99, "max_ms": 74.085, "p95_ms": 67.835}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.531, "count": 99, "max_ms": 94.172, "p95_ms": 89.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 243.111, "count": 100, "max_ms": 328.216, "p95_ms": 278.863}, "read_interval": {"avg_ms": 252.966, "count": 99, "max_ms": 338.715, "p95_ms": 291.404}, "triton_wait": {"avg_ms": 66.689, "count": 99, "max_ms": 94.872, "p95_ms": 90.067}, "visualize": {"avg_ms": 11.431, "count": 99, "max_ms": 16.236, "p95_ms": 12.419}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 608}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4053.941, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562185432340] error while decoding MB 4 7, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 130, "h264_decode_error_count": 135, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:48:56+09:00", "last_error_summary": "[h264 @ 0x562185432340] error while decoding MB 4 7, bytestream -5", "last_exit_at": "2026-07-01T19:47:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.035, "last_started_at": "2026-07-01T19:47:06+09:00", "pid": 20074, "process": {"exists": false, "pid": 20074}, "restart_count": 153, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 154}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.938, "sample_count": 1160, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73071, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:48:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.045, "last_frame_pushed_at": "2026-07-01T19:48:56+09:00", "last_frame_read_age_sec": 0.038, "last_frame_read_at": "2026-07-01T19:48:56+09:00", "mjpeg": {"age_sec": 0.026, "exists": true, "mtime": "2026-07-01T19:48:56.586643+09:00", "mtime_epoch": 1782902936.586643, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171773}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:48:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 902.129, "name": "celery", "pid": 237, "rss_mb": 896.973, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.046, "pushed": 4.046, "read": 4.046}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.497, "count": 120, "max_ms": 12.218, "p95_ms": 10.456}, "gstreamer_push": {"avg_ms": 0.911, "count": 121, "max_ms": 1.967, "p95_ms": 0.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.23, "count": 120, "max_ms": 70.322, "p95_ms": 62.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.178, "count": 120, "max_ms": 94.671, "p95_ms": 86.598}, "yolo11_640_fire_11class": {"avg_ms": 53.587, "count": 120, "max_ms": 71.726, "p95_ms": 67.808}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.195, "count": 120, "max_ms": 93.702, "p95_ms": 90.053}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.515, "count": 121, "max_ms": 315.59, "p95_ms": 276.938}, "read_interval": {"avg_ms": 249.048, "count": 121, "max_ms": 326.092, "p95_ms": 286.366}, "triton_wait": {"avg_ms": 68.939, "count": 120, "max_ms": 94.683, "p95_ms": 90.065}, "visualize": {"avg_ms": 11.105, "count": 120, "max_ms": 14.734, "p95_ms": 12.122}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 608}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4053.332, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:48:56+09:00", "last_error_summary": "[h264 @ 0x562185432340] error while decoding MB 4 7, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 130, "h264_decode_error_count": 135, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:48:56+09:00", "last_error_summary": "[h264 @ 0x562185432340] error while decoding MB 4 7, bytestream -5", "last_exit_at": "2026-07-01T19:48:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.212, "last_started_at": "2026-07-01T19:47:06+09:00", "pid": 20074, "process": {"exists": false, "pid": 20074}, "restart_count": 153, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 154}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.938, "sample_count": 1160, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73072, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:48:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:48:56+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T19:48:56+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-01T19:48:56.586643+09:00", "mtime_epoch": 1782902936.586643, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171773}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:48:56+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 902.129, "name": "celery", "pid": 237, "rss_mb": 896.973, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.061, "pushed": 4.061, "read": 4.027}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.49, "count": 121, "max_ms": 12.218, "p95_ms": 10.456}, "gstreamer_push": {"avg_ms": 0.911, "count": 121, "max_ms": 1.967, "p95_ms": 0.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.144, "count": 121, "max_ms": 70.322, "p95_ms": 62.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.092, "count": 121, "max_ms": 94.671, "p95_ms": 86.598}, "yolo11_640_fire_11class": {"avg_ms": 53.522, "count": 121, "max_ms": 71.726, "p95_ms": 67.808}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.116, "count": 121, "max_ms": 93.702, "p95_ms": 90.053}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.027, "count": 121, "max_ms": 315.59, "p95_ms": 276.938}, "read_interval": {"avg_ms": 248.806, "count": 120, "max_ms": 326.092, "p95_ms": 286.366}, "triton_wait": {"avg_ms": 68.854, "count": 121, "max_ms": 94.683, "p95_ms": 90.065}, "visualize": {"avg_ms": 11.093, "count": 121, "max_ms": 14.734, "p95_ms": 12.122}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.796, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 608}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4053.324, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 130, "h264_decode_error_count": 135, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:48:56+09:00", "last_error_summary": "[h264 @ 0x562185432340] error while decoding MB 4 7, bytestream -5", "last_exit_at": "2026-07-01T19:48:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.215, "last_started_at": "2026-07-01T19:47:06+09:00", "pid": 20074, "process": {"exists": false, "pid": 20074}, "restart_count": 153, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 154}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.938, "sample_count": 1160, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73072, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:48:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:48:56+09:00", "last_frame_read_age_sec": 0.217, "last_frame_read_at": "2026-07-01T19:48:56+09:00", "mjpeg": {"age_sec": 0.206, "exists": true, "mtime": "2026-07-01T19:48:56.586643+09:00", "mtime_epoch": 1782902936.586643, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171773}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:48:56+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 902.129, "name": "celery", "pid": 237, "rss_mb": 896.973, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.06, "pushed": 4.06, "read": 4.027}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.49, "count": 121, "max_ms": 12.218, "p95_ms": 10.456}, "gstreamer_push": {"avg_ms": 0.911, "count": 121, "max_ms": 1.967, "p95_ms": 0.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.144, "count": 121, "max_ms": 70.322, "p95_ms": 62.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.092, "count": 121, "max_ms": 94.671, "p95_ms": 86.598}, "yolo11_640_fire_11class": {"avg_ms": 53.522, "count": 121, "max_ms": 71.726, "p95_ms": 67.808}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.116, "count": 121, "max_ms": 93.702, "p95_ms": 90.053}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.027, "count": 121, "max_ms": 315.59, "p95_ms": 276.938}, "read_interval": {"avg_ms": 248.806, "count": 120, "max_ms": 326.092, "p95_ms": 286.366}, "triton_wait": {"avg_ms": 68.854, "count": 121, "max_ms": 94.683, "p95_ms": 90.065}, "visualize": {"avg_ms": 11.093, "count": 121, "max_ms": 14.734, "p95_ms": 12.122}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.799, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 608}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4160.848, "limit_mb": 8796093022207.996}, "detail": {"pid": 20181, "restart_count": 154, "start_count": 155}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 130, "h264_decode_error_count": 135, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:48:56+09:00", "last_error_summary": "[h264 @ 0x562185432340] error while decoding MB 4 7, bytestream -5", "last_exit_at": "2026-07-01T19:48:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:48:59+09:00", "pid": 20181, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.387, "name": "ffmpeg", "pid": 20181, "rss_mb": 139.387, "state": "S (sleeping)", "threads": 106}, "restart_count": 154, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 155}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.938, "sample_count": 1141, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73072, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:48:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.942, "last_frame_pushed_at": "2026-07-01T19:48:56+09:00", "last_frame_read_age_sec": 5.153, "last_frame_read_at": "2026-07-01T19:48:56+09:00", "mjpeg": {"age_sec": 4.926, "exists": true, "mtime": "2026-07-01T19:48:56.802646+09:00", "mtime_epoch": 1782902936.8026464, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164355}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:49:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 902.129, "name": "celery", "pid": 237, "rss_mb": 896.973, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.435, "pushed": 3.435, "read": 3.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.39, "count": 102, "max_ms": 10.473, "p95_ms": 10.157}, "gstreamer_push": {"avg_ms": 0.909, "count": 103, "max_ms": 1.828, "p95_ms": 0.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.435, "count": 102, "max_ms": 65.643, "p95_ms": 60.788}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.096, "count": 102, "max_ms": 94.671, "p95_ms": 86.561}, "yolo11_640_fire_11class": {"avg_ms": 52.843, "count": 102, "max_ms": 70.115, "p95_ms": 66.155}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.087, "count": 102, "max_ms": 93.702, "p95_ms": 89.1}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.29, "count": 103, "max_ms": 315.59, "p95_ms": 268.958}, "read_interval": {"avg_ms": 246.064, "count": 102, "max_ms": 326.092, "p95_ms": 279.32}, "triton_wait": {"avg_ms": 67.885, "count": 102, "max_ms": 94.683, "p95_ms": 89.263}, "visualize": {"avg_ms": 10.988, "count": 102, "max_ms": 12.392, "p95_ms": 12.003}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 612}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4054.578, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e5ffe9ad40] error while decoding MB 2 32, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 131, "h264_decode_error_count": 136, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:50:49+09:00", "last_error_summary": "[h264 @ 0x55e5ffe9ad40] error while decoding MB 2 32, bytestream -5", "last_exit_at": "2026-07-01T19:48:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.745, "last_started_at": "2026-07-01T19:48:59+09:00", "pid": 20181, "process": {"exists": false, "pid": 20181}, "restart_count": 154, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 155}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.231, "sample_count": 1164, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73504, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:50:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T19:50:49+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T19:50:49+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T19:50:49.428391+09:00", "mtime_epoch": 1782903049.4283915, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172495}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:50:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 902.129, "name": "celery", "pid": 237, "rss_mb": 896.621, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.036, "pushed": 4.036, "read": 4.036}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.424, "count": 120, "max_ms": 14.182, "p95_ms": 10.666}, "gstreamer_push": {"avg_ms": 0.91, "count": 121, "max_ms": 2.263, "p95_ms": 0.994}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.148, "count": 120, "max_ms": 81.899, "p95_ms": 61.248}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.589, "count": 120, "max_ms": 97.165, "p95_ms": 91.956}, "yolo11_640_fire_11class": {"avg_ms": 53.804, "count": 120, "max_ms": 82.476, "p95_ms": 68.767}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.106, "count": 120, "max_ms": 99.337, "p95_ms": 91.168}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.552, "count": 121, "max_ms": 331.439, "p95_ms": 277.874}, "read_interval": {"avg_ms": 249.232, "count": 121, "max_ms": 344.804, "p95_ms": 287.523}, "triton_wait": {"avg_ms": 69.299, "count": 120, "max_ms": 99.349, "p95_ms": 91.969}, "visualize": {"avg_ms": 11.314, "count": 120, "max_ms": 15.57, "p95_ms": 13.085}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 612}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4053.789, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:50:49+09:00", "last_error_summary": "[h264 @ 0x55e5ffe9ad40] error while decoding MB 2 32, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 131, "h264_decode_error_count": 136, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:50:49+09:00", "last_error_summary": "[h264 @ 0x55e5ffe9ad40] error while decoding MB 2 32, bytestream -5", "last_exit_at": "2026-07-01T19:50:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.95, "last_started_at": "2026-07-01T19:48:59+09:00", "pid": 20181, "process": {"exists": false, "pid": 20181}, "restart_count": 154, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 155}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.231, "sample_count": 1164, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73505, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:50:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:50:49+09:00", "last_frame_read_age_sec": 0.254, "last_frame_read_at": "2026-07-01T19:50:49+09:00", "mjpeg": {"age_sec": 0.249, "exists": true, "mtime": "2026-07-01T19:50:49.428391+09:00", "mtime_epoch": 1782903049.4283915, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172495}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:50:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 902.129, "name": "celery", "pid": 237, "rss_mb": 896.621, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.04, "pushed": 4.04, "read": 4.006}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.414, "count": 120, "max_ms": 13.013, "p95_ms": 10.666}, "gstreamer_push": {"avg_ms": 0.909, "count": 121, "max_ms": 2.263, "p95_ms": 0.994}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.211, "count": 120, "max_ms": 81.899, "p95_ms": 61.248}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.78, "count": 120, "max_ms": 97.165, "p95_ms": 91.956}, "yolo11_640_fire_11class": {"avg_ms": 53.879, "count": 120, "max_ms": 82.476, "p95_ms": 68.767}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.27, "count": 120, "max_ms": 99.337, "p95_ms": 91.168}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.768, "count": 121, "max_ms": 331.439, "p95_ms": 277.874}, "read_interval": {"avg_ms": 249.363, "count": 120, "max_ms": 344.804, "p95_ms": 287.523}, "triton_wait": {"avg_ms": 69.469, "count": 120, "max_ms": 99.349, "p95_ms": 91.969}, "visualize": {"avg_ms": 11.277, "count": 120, "max_ms": 15.57, "p95_ms": 12.661}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 612}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4053.723, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 131, "h264_decode_error_count": 136, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:50:49+09:00", "last_error_summary": "[h264 @ 0x55e5ffe9ad40] error while decoding MB 2 32, bytestream -5", "last_exit_at": "2026-07-01T19:50:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.953, "last_started_at": "2026-07-01T19:48:59+09:00", "pid": 20181, "process": {"exists": false, "pid": 20181}, "restart_count": 154, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 155}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.231, "sample_count": 1164, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73505, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:50:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:50:49+09:00", "last_frame_read_age_sec": 0.258, "last_frame_read_at": "2026-07-01T19:50:49+09:00", "mjpeg": {"age_sec": 0.253, "exists": true, "mtime": "2026-07-01T19:50:49.428391+09:00", "mtime_epoch": 1782903049.4283915, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172495}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:50:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 902.129, "name": "celery", "pid": 237, "rss_mb": 896.621, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.039, "pushed": 4.039, "read": 4.006}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.414, "count": 120, "max_ms": 13.013, "p95_ms": 10.666}, "gstreamer_push": {"avg_ms": 0.909, "count": 121, "max_ms": 2.263, "p95_ms": 0.994}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.211, "count": 120, "max_ms": 81.899, "p95_ms": 61.248}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.78, "count": 120, "max_ms": 97.165, "p95_ms": 91.956}, "yolo11_640_fire_11class": {"avg_ms": 53.879, "count": 120, "max_ms": 82.476, "p95_ms": 68.767}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.27, "count": 120, "max_ms": 99.337, "p95_ms": 91.168}}, "truncated_count": 0}, "processing_total": {"avg_ms": 239.768, "count": 121, "max_ms": 331.439, "p95_ms": 277.874}, "read_interval": {"avg_ms": 249.363, "count": 120, "max_ms": 344.804, "p95_ms": 287.523}, "triton_wait": {"avg_ms": 69.469, "count": 120, "max_ms": 99.349, "p95_ms": 91.969}, "visualize": {"avg_ms": 11.277, "count": 120, "max_ms": 15.57, "p95_ms": 12.661}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 612}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4159.559, "limit_mb": 8796093022207.996}, "detail": {"pid": 20288, "restart_count": 155, "start_count": 156}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 131, "h264_decode_error_count": 136, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:50:49+09:00", "last_error_summary": "[h264 @ 0x55e5ffe9ad40] error while decoding MB 2 32, bytestream -5", "last_exit_at": "2026-07-01T19:50:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:50:51+09:00", "pid": 20288, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.691, "name": "ffmpeg", "pid": 20288, "rss_mb": 136.691, "state": "S (sleeping)", "threads": 106}, "restart_count": 155, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 156}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.231, "sample_count": 1148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73505, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:50:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.325, "last_frame_pushed_at": "2026-07-01T19:50:49+09:00", "last_frame_read_age_sec": 4.577, "last_frame_read_at": "2026-07-01T19:50:49+09:00", "mjpeg": {"age_sec": 4.312, "exists": true, "mtime": "2026-07-01T19:50:49.688396+09:00", "mtime_epoch": 1782903049.6883955, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172733}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:50:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 902.129, "name": "celery", "pid": 237, "rss_mb": 896.621, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.455, "pushed": 3.455, "read": 3.422}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.425, "count": 103, "max_ms": 13.013, "p95_ms": 10.373}, "gstreamer_push": {"avg_ms": 0.901, "count": 103, "max_ms": 2.263, "p95_ms": 0.92}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.604, "count": 103, "max_ms": 81.899, "p95_ms": 63.85}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.49, "count": 103, "max_ms": 97.165, "p95_ms": 92.582}, "yolo11_640_fire_11class": {"avg_ms": 54.156, "count": 103, "max_ms": 82.476, "p95_ms": 68.973}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.776, "count": 103, "max_ms": 99.337, "p95_ms": 91.286}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.137, "count": 103, "max_ms": 331.439, "p95_ms": 277.874}, "read_interval": {"avg_ms": 249.675, "count": 102, "max_ms": 344.804, "p95_ms": 287.195}, "triton_wait": {"avg_ms": 70.072, "count": 103, "max_ms": 99.349, "p95_ms": 92.595}, "visualize": {"avg_ms": 11.249, "count": 103, "max_ms": 15.57, "p95_ms": 12.641}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.81, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4061.086, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556673831880] error while decoding MB 70 20, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 132, "h264_decode_error_count": 137, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:52:42+09:00", "last_error_summary": "[h264 @ 0x556673831880] error while decoding MB 70 20, bytestream -6", "last_exit_at": "2026-07-01T19:50:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.902, "last_started_at": "2026-07-01T19:50:51+09:00", "pid": 20288, "process": {"exists": false, "pid": 20288}, "restart_count": 155, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 156}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.669, "sample_count": 1163, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73937, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:52:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T19:52:42+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T19:52:42+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-01T19:52:42.858145+09:00", "mtime_epoch": 1782903162.8581455, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174114}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:52:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.539, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 119, "pushed_frames": 119, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.996, "pushed": 3.996, "read": 3.996}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.611, "count": 119, "max_ms": 14.629, "p95_ms": 12.205}, "gstreamer_push": {"avg_ms": 0.893, "count": 119, "max_ms": 1.896, "p95_ms": 0.968}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.662, "count": 119, "max_ms": 70.603, "p95_ms": 67.846}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.073, "count": 119, "max_ms": 91.467, "p95_ms": 86.815}, "yolo11_640_fire_11class": {"avg_ms": 54.253, "count": 119, "max_ms": 73.187, "p95_ms": 68.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.629, "count": 119, "max_ms": 91.794, "p95_ms": 90.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.518, "count": 119, "max_ms": 341.001, "p95_ms": 275.651}, "read_interval": {"avg_ms": 251.977, "count": 119, "max_ms": 351.599, "p95_ms": 286.293}, "triton_wait": {"avg_ms": 70.202, "count": 119, "max_ms": 91.806, "p95_ms": 90.082}, "visualize": {"avg_ms": 11.363, "count": 119, "max_ms": 17.291, "p95_ms": 14.679}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.778, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4060.219, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:52:42+09:00", "last_error_summary": "[h264 @ 0x556673831880] error while decoding MB 70 20, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 132, "h264_decode_error_count": 137, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:52:42+09:00", "last_error_summary": "[h264 @ 0x556673831880] error while decoding MB 70 20, bytestream -6", "last_exit_at": "2026-07-01T19:52:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.106, "last_started_at": "2026-07-01T19:50:51+09:00", "pid": 20288, "process": {"exists": false, "pid": 20288}, "restart_count": 155, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 156}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.669, "sample_count": 1163, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73938, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:52:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T19:52:43+09:00", "last_frame_read_age_sec": 0.256, "last_frame_read_at": "2026-07-01T19:52:42+09:00", "mjpeg": {"age_sec": 0.247, "exists": true, "mtime": "2026-07-01T19:52:42.858145+09:00", "mtime_epoch": 1782903162.8581455, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174114}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:52:43+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.539, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.003, "pushed": 4.003, "read": 3.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.612, "count": 119, "max_ms": 14.629, "p95_ms": 12.205}, "gstreamer_push": {"avg_ms": 0.893, "count": 120, "max_ms": 1.896, "p95_ms": 0.931}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.546, "count": 119, "max_ms": 70.603, "p95_ms": 67.846}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.999, "count": 119, "max_ms": 91.467, "p95_ms": 86.815}, "yolo11_640_fire_11class": {"avg_ms": 54.179, "count": 119, "max_ms": 73.187, "p95_ms": 68.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.529, "count": 119, "max_ms": 91.794, "p95_ms": 90.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.584, "count": 120, "max_ms": 341.001, "p95_ms": 275.155}, "read_interval": {"avg_ms": 251.977, "count": 119, "max_ms": 351.599, "p95_ms": 286.293}, "triton_wait": {"avg_ms": 70.121, "count": 119, "max_ms": 91.806, "p95_ms": 90.082}, "visualize": {"avg_ms": 11.366, "count": 119, "max_ms": 17.291, "p95_ms": 14.679}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4060.336, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 132, "h264_decode_error_count": 137, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:52:42+09:00", "last_error_summary": "[h264 @ 0x556673831880] error while decoding MB 70 20, bytestream -6", "last_exit_at": "2026-07-01T19:52:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.109, "last_started_at": "2026-07-01T19:50:51+09:00", "pid": 20288, "process": {"exists": false, "pid": 20288}, "restart_count": 155, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 156}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.669, "sample_count": 1163, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73938, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:52:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T19:52:43+09:00", "last_frame_read_age_sec": 0.259, "last_frame_read_at": "2026-07-01T19:52:42+09:00", "mjpeg": {"age_sec": 0.25, "exists": true, "mtime": "2026-07-01T19:52:42.858145+09:00", "mtime_epoch": 1782903162.8581455, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174114}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:52:43+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.539, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 120, "pushed_frames": 120, "read_frames": 119, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.002, "pushed": 4.002, "read": 3.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.612, "count": 119, "max_ms": 14.629, "p95_ms": 12.205}, "gstreamer_push": {"avg_ms": 0.893, "count": 120, "max_ms": 1.896, "p95_ms": 0.931}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.546, "count": 119, "max_ms": 70.603, "p95_ms": 67.846}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.999, "count": 119, "max_ms": 91.467, "p95_ms": 86.815}, "yolo11_640_fire_11class": {"avg_ms": 54.179, "count": 119, "max_ms": 73.187, "p95_ms": 68.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.529, "count": 119, "max_ms": 91.794, "p95_ms": 90.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 242.584, "count": 120, "max_ms": 341.001, "p95_ms": 275.155}, "read_interval": {"avg_ms": 251.977, "count": 119, "max_ms": 351.599, "p95_ms": 286.293}, "triton_wait": {"avg_ms": 70.121, "count": 119, "max_ms": 91.806, "p95_ms": 90.082}, "visualize": {"avg_ms": 11.366, "count": 119, "max_ms": 17.291, "p95_ms": 14.679}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4162.273, "limit_mb": 8796093022207.996}, "detail": {"pid": 20395, "restart_count": 156, "start_count": 157}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 132, "h264_decode_error_count": 137, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:52:42+09:00", "last_error_summary": "[h264 @ 0x556673831880] error while decoding MB 70 20, bytestream -6", "last_exit_at": "2026-07-01T19:52:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:52:45+09:00", "pid": 20395, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.312, "name": "ffmpeg", "pid": 20395, "rss_mb": 136.312, "state": "S (sleeping)", "threads": 106}, "restart_count": 156, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 157}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 276.669, "sample_count": 1145, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73938, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:52:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.829, "last_frame_pushed_at": "2026-07-01T19:52:43+09:00", "last_frame_read_age_sec": 5.081, "last_frame_read_at": "2026-07-01T19:52:42+09:00", "mjpeg": {"age_sec": 4.812, "exists": true, "mtime": "2026-07-01T19:52:43.118150+09:00", "mtime_epoch": 1782903163.1181495, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 174306}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:52:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.539, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.382, "pushed": 3.382, "read": 3.348}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.673, "count": 100, "max_ms": 14.629, "p95_ms": 12.205}, "gstreamer_push": {"avg_ms": 0.889, "count": 101, "max_ms": 1.896, "p95_ms": 0.923}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.815, "count": 100, "max_ms": 70.603, "p95_ms": 63.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.427, "count": 100, "max_ms": 91.467, "p95_ms": 86.455}, "yolo11_640_fire_11class": {"avg_ms": 53.423, "count": 100, "max_ms": 73.187, "p95_ms": 67.525}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.488, "count": 100, "max_ms": 91.437, "p95_ms": 88.627}}, "truncated_count": 0}, "processing_total": {"avg_ms": 240.823, "count": 101, "max_ms": 303.212, "p95_ms": 275.155}, "read_interval": {"avg_ms": 250.167, "count": 100, "max_ms": 312.221, "p95_ms": 284.146}, "triton_wait": {"avg_ms": 69.183, "count": 100, "max_ms": 91.48, "p95_ms": 88.639}, "visualize": {"avg_ms": 11.365, "count": 100, "max_ms": 17.291, "p95_ms": 14.679}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 619}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4056.148, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56027ef4c640] error while decoding MB 17 30, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 133, "h264_decode_error_count": 138, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:54:34+09:00", "last_error_summary": "[h264 @ 0x56027ef4c640] error while decoding MB 17 30, bytestream -5", "last_exit_at": "2026-07-01T19:52:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.937, "last_started_at": "2026-07-01T19:52:45+09:00", "pid": 20395, "process": {"exists": false, "pid": 20395}, "restart_count": 156, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 157}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.145, "sample_count": 1173, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74376, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:54:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T19:54:34+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T19:54:34+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-01T19:54:34.827874+09:00", "mtime_epoch": 1782903274.8278744, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172162}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:54:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.207, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.153, "pushed": 4.153, "read": 4.153}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.056, "count": 123, "max_ms": 10.833, "p95_ms": 9.628}, "gstreamer_push": {"avg_ms": 0.93, "count": 124, "max_ms": 2.737, "p95_ms": 1.778}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.833, "count": 123, "max_ms": 71.861, "p95_ms": 56.797}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.136, "count": 123, "max_ms": 93.15, "p95_ms": 83.967}, "yolo11_640_fire_11class": {"avg_ms": 52.113, "count": 123, "max_ms": 71.304, "p95_ms": 61.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.596, "count": 123, "max_ms": 92.094, "p95_ms": 78.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.862, "count": 124, "max_ms": 304.731, "p95_ms": 277.073}, "read_interval": {"avg_ms": 242.512, "count": 124, "max_ms": 315.295, "p95_ms": 289.493}, "triton_wait": {"avg_ms": 67.324, "count": 123, "max_ms": 93.164, "p95_ms": 83.979}, "visualize": {"avg_ms": 10.901, "count": 123, "max_ms": 13.075, "p95_ms": 11.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.858, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 619}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4055.453, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:54:34+09:00", "last_error_summary": "[h264 @ 0x56027ef4c640] error while decoding MB 17 30, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 133, "h264_decode_error_count": 138, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:54:34+09:00", "last_error_summary": "[h264 @ 0x56027ef4c640] error while decoding MB 17 30, bytestream -5", "last_exit_at": "2026-07-01T19:54:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.115, "last_started_at": "2026-07-01T19:52:45+09:00", "pid": 20395, "process": {"exists": false, "pid": 20395}, "restart_count": 156, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 157}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.073, "sample_count": 1173, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:54:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:54:35+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-01T19:54:34+09:00", "mjpeg": {"age_sec": 0.216, "exists": true, "mtime": "2026-07-01T19:54:34.827874+09:00", "mtime_epoch": 1782903274.8278744, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172162}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:54:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.207, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.162, "pushed": 4.162, "read": 4.128}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.056, "count": 124, "max_ms": 10.833, "p95_ms": 9.628}, "gstreamer_push": {"avg_ms": 0.926, "count": 124, "max_ms": 2.737, "p95_ms": 1.778}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.801, "count": 124, "max_ms": 71.861, "p95_ms": 56.797}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.125, "count": 124, "max_ms": 93.15, "p95_ms": 83.967}, "yolo11_640_fire_11class": {"avg_ms": 52.102, "count": 124, "max_ms": 71.304, "p95_ms": 61.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.569, "count": 124, "max_ms": 92.094, "p95_ms": 78.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.539, "count": 124, "max_ms": 304.731, "p95_ms": 277.073}, "read_interval": {"avg_ms": 242.192, "count": 123, "max_ms": 315.295, "p95_ms": 289.493}, "triton_wait": {"avg_ms": 67.312, "count": 124, "max_ms": 93.164, "p95_ms": 83.979}, "visualize": {"avg_ms": 10.898, "count": 124, "max_ms": 13.075, "p95_ms": 11.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.797, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 619}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4055.445, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 133, "h264_decode_error_count": 138, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:54:34+09:00", "last_error_summary": "[h264 @ 0x56027ef4c640] error while decoding MB 17 30, bytestream -5", "last_exit_at": "2026-07-01T19:54:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.118, "last_started_at": "2026-07-01T19:52:45+09:00", "pid": 20395, "process": {"exists": false, "pid": 20395}, "restart_count": 156, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 157}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.073, "sample_count": 1173, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:54:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:54:35+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T19:54:34+09:00", "mjpeg": {"age_sec": 0.219, "exists": true, "mtime": "2026-07-01T19:54:34.827874+09:00", "mtime_epoch": 1782903274.8278744, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172162}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:54:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.207, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.161, "pushed": 4.161, "read": 4.127}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.056, "count": 124, "max_ms": 10.833, "p95_ms": 9.628}, "gstreamer_push": {"avg_ms": 0.926, "count": 124, "max_ms": 2.737, "p95_ms": 1.778}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.801, "count": 124, "max_ms": 71.861, "p95_ms": 56.797}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.125, "count": 124, "max_ms": 93.15, "p95_ms": 83.967}, "yolo11_640_fire_11class": {"avg_ms": 52.102, "count": 124, "max_ms": 71.304, "p95_ms": 61.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.569, "count": 124, "max_ms": 92.094, "p95_ms": 78.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.539, "count": 124, "max_ms": 304.731, "p95_ms": 277.073}, "read_interval": {"avg_ms": 242.192, "count": 123, "max_ms": 315.295, "p95_ms": 289.493}, "triton_wait": {"avg_ms": 67.312, "count": 124, "max_ms": 93.164, "p95_ms": 83.979}, "visualize": {"avg_ms": 10.898, "count": 124, "max_ms": 13.075, "p95_ms": 11.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.8, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 619}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4163.652, "limit_mb": 8796093022207.996}, "detail": {"pid": 20502, "restart_count": 157, "start_count": 158}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 133, "h264_decode_error_count": 138, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:54:34+09:00", "last_error_summary": "[h264 @ 0x56027ef4c640] error while decoding MB 17 30, bytestream -5", "last_exit_at": "2026-07-01T19:54:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:54:37+09:00", "pid": 20502, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.691, "name": "ffmpeg", "pid": 20502, "rss_mb": 139.691, "state": "S (sleeping)", "threads": 106}, "restart_count": 157, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 158}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 277.145, "sample_count": 1156, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:54:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.054, "last_frame_pushed_at": "2026-07-01T19:54:35+09:00", "last_frame_read_age_sec": 4.282, "last_frame_read_at": "2026-07-01T19:54:34+09:00", "mjpeg": {"age_sec": 4.035, "exists": true, "mtime": "2026-07-01T19:54:35.059878+09:00", "mtime_epoch": 1782903275.0598779, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171003}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:54:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.207, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.6, "pushed": 3.6, "read": 3.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.97, "count": 107, "max_ms": 10.833, "p95_ms": 9.551}, "gstreamer_push": {"avg_ms": 0.927, "count": 108, "max_ms": 2.737, "p95_ms": 1.778}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.094, "count": 107, "max_ms": 71.861, "p95_ms": 56.979}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.406, "count": 107, "max_ms": 93.15, "p95_ms": 83.967}, "yolo11_640_fire_11class": {"avg_ms": 52.208, "count": 107, "max_ms": 71.304, "p95_ms": 61.213}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.729, "count": 107, "max_ms": 92.094, "p95_ms": 78.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.796, "count": 108, "max_ms": 304.731, "p95_ms": 277.073}, "read_interval": {"avg_ms": 242.461, "count": 107, "max_ms": 315.295, "p95_ms": 289.493}, "triton_wait": {"avg_ms": 67.579, "count": 107, "max_ms": 93.164, "p95_ms": 83.979}, "visualize": {"avg_ms": 10.927, "count": 107, "max_ms": 13.075, "p95_ms": 11.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 623}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4059.668, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f295217f00] error while decoding MB 68 27, bytestream -24", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 134, "h264_decode_error_count": 139, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:56:27+09:00", "last_error_summary": "[h264 @ 0x55f295217f00] error while decoding MB 68 27, bytestream -24", "last_exit_at": "2026-07-01T19:54:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.618, "last_started_at": "2026-07-01T19:54:37+09:00", "pid": 20502, "process": {"exists": false, "pid": 20502}, "restart_count": 157, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 158}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.77, "sample_count": 1194, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74830, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:56:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T19:56:27+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T19:56:27+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T19:56:27.669615+09:00", "mtime_epoch": 1782903387.6696146, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176914}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:56:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.117, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.256, "pushed": 4.256, "read": 4.256}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.813, "count": 127, "max_ms": 10.929, "p95_ms": 9.847}, "gstreamer_push": {"avg_ms": 0.917, "count": 127, "max_ms": 2.847, "p95_ms": 1.301}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.2, "count": 127, "max_ms": 74.485, "p95_ms": 58.643}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.821, "count": 127, "max_ms": 97.58, "p95_ms": 83.655}, "yolo11_640_fire_11class": {"avg_ms": 53.241, "count": 127, "max_ms": 75.01, "p95_ms": 63.945}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.752, "count": 127, "max_ms": 96.031, "p95_ms": 82.055}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.874, "count": 127, "max_ms": 311.34, "p95_ms": 252.967}, "read_interval": {"avg_ms": 236.372, "count": 127, "max_ms": 322.949, "p95_ms": 261.654}, "triton_wait": {"avg_ms": 68.954, "count": 127, "max_ms": 97.591, "p95_ms": 85.024}, "visualize": {"avg_ms": 10.51, "count": 127, "max_ms": 12.632, "p95_ms": 11.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.84, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 623}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4058.797, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:56:27+09:00", "last_error_summary": "[h264 @ 0x55f295217f00] error while decoding MB 68 27, bytestream -24", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 134, "h264_decode_error_count": 139, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:56:27+09:00", "last_error_summary": "[h264 @ 0x55f295217f00] error while decoding MB 68 27, bytestream -24", "last_exit_at": "2026-07-01T19:56:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.8, "last_started_at": "2026-07-01T19:54:37+09:00", "pid": 20502, "process": {"exists": false, "pid": 20502}, "restart_count": 157, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 158}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.77, "sample_count": 1195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74831, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:56:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:56:27+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-01T19:56:27+09:00", "mjpeg": {"age_sec": 0.225, "exists": true, "mtime": "2026-07-01T19:56:27.669615+09:00", "mtime_epoch": 1782903387.6696146, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176914}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:56:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.117, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.265, "pushed": 4.265, "read": 4.231}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.809, "count": 127, "max_ms": 10.929, "p95_ms": 9.847}, "gstreamer_push": {"avg_ms": 0.917, "count": 127, "max_ms": 2.847, "p95_ms": 1.301}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.197, "count": 127, "max_ms": 74.485, "p95_ms": 58.643}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.987, "count": 127, "max_ms": 97.58, "p95_ms": 85.012}, "yolo11_640_fire_11class": {"avg_ms": 53.367, "count": 127, "max_ms": 75.01, "p95_ms": 64.46}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.907, "count": 127, "max_ms": 96.031, "p95_ms": 82.219}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.853, "count": 127, "max_ms": 311.34, "p95_ms": 252.967}, "read_interval": {"avg_ms": 236.35, "count": 126, "max_ms": 322.949, "p95_ms": 261.654}, "triton_wait": {"avg_ms": 69.119, "count": 127, "max_ms": 97.591, "p95_ms": 85.66}, "visualize": {"avg_ms": 10.501, "count": 127, "max_ms": 12.632, "p95_ms": 11.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.777, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 623}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4058.789, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 134, "h264_decode_error_count": 139, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:56:27+09:00", "last_error_summary": "[h264 @ 0x55f295217f00] error while decoding MB 68 27, bytestream -24", "last_exit_at": "2026-07-01T19:56:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.803, "last_started_at": "2026-07-01T19:54:37+09:00", "pid": 20502, "process": {"exists": false, "pid": 20502}, "restart_count": 157, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 158}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.77, "sample_count": 1195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74831, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:56:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:56:27+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-01T19:56:27+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-01T19:56:27.669615+09:00", "mtime_epoch": 1782903387.6696146, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176914}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:56:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.117, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.265, "pushed": 4.265, "read": 4.231}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.809, "count": 127, "max_ms": 10.929, "p95_ms": 9.847}, "gstreamer_push": {"avg_ms": 0.917, "count": 127, "max_ms": 2.847, "p95_ms": 1.301}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.197, "count": 127, "max_ms": 74.485, "p95_ms": 58.643}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.987, "count": 127, "max_ms": 97.58, "p95_ms": 85.012}, "yolo11_640_fire_11class": {"avg_ms": 53.367, "count": 127, "max_ms": 75.01, "p95_ms": 64.46}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.907, "count": 127, "max_ms": 96.031, "p95_ms": 82.219}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.853, "count": 127, "max_ms": 311.34, "p95_ms": 252.967}, "read_interval": {"avg_ms": 236.35, "count": 126, "max_ms": 322.949, "p95_ms": 261.654}, "triton_wait": {"avg_ms": 69.119, "count": 127, "max_ms": 97.591, "p95_ms": 85.66}, "visualize": {"avg_ms": 10.501, "count": 127, "max_ms": 12.632, "p95_ms": 11.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.78, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 623}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4165.617, "limit_mb": 8796093022207.996}, "detail": {"pid": 20609, "restart_count": 158, "start_count": 159}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 134, "h264_decode_error_count": 139, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:56:27+09:00", "last_error_summary": "[h264 @ 0x55f295217f00] error while decoding MB 68 27, bytestream -24", "last_exit_at": "2026-07-01T19:56:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:56:30+09:00", "pid": 20609, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.605, "name": "ffmpeg", "pid": 20609, "rss_mb": 136.605, "state": "S (sleeping)", "threads": 106}, "restart_count": 158, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 159}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.77, "sample_count": 1176, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74831, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:56:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.613, "last_frame_pushed_at": "2026-07-01T19:56:27+09:00", "last_frame_read_age_sec": 4.842, "last_frame_read_at": "2026-07-01T19:56:27+09:00", "mjpeg": {"age_sec": 4.606, "exists": true, "mtime": "2026-07-01T19:56:27.897618+09:00", "mtime_epoch": 1782903387.897618, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172919}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:56:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.117, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.585, "pushed": 3.585, "read": 3.552}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.815, "count": 107, "max_ms": 10.929, "p95_ms": 9.847}, "gstreamer_push": {"avg_ms": 0.928, "count": 107, "max_ms": 2.847, "p95_ms": 1.754}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.746, "count": 107, "max_ms": 74.485, "p95_ms": 59.646}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.421, "count": 107, "max_ms": 97.58, "p95_ms": 88.115}, "yolo11_640_fire_11class": {"avg_ms": 53.693, "count": 107, "max_ms": 75.01, "p95_ms": 65.667}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.066, "count": 107, "max_ms": 96.031, "p95_ms": 83.308}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.471, "count": 107, "max_ms": 311.34, "p95_ms": 253.135}, "read_interval": {"avg_ms": 237.887, "count": 106, "max_ms": 322.949, "p95_ms": 262.126}, "triton_wait": {"avg_ms": 69.541, "count": 107, "max_ms": 97.591, "p95_ms": 88.127}, "visualize": {"avg_ms": 10.524, "count": 107, "max_ms": 12.632, "p95_ms": 11.222}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.842, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 627}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4057.844, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55abfbf33580] error while decoding MB 79 38, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 135, "h264_decode_error_count": 140, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:58:18+09:00", "last_error_summary": "[h264 @ 0x55abfbf33580] error while decoding MB 79 38, bytestream -5", "last_exit_at": "2026-07-01T19:56:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.434, "last_started_at": "2026-07-01T19:56:30+09:00", "pid": 20609, "process": {"exists": false, "pid": 20609}, "restart_count": 158, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 159}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 262.78, "sample_count": 1215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75279, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:58:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T19:58:18+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T19:58:18+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T19:58:18.895328+09:00", "mtime_epoch": 1782903498.895328, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178147}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:58:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.148, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.285, "pushed": 4.285, "read": 4.285}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.783, "count": 127, "max_ms": 12.671, "p95_ms": 9.771}, "gstreamer_push": {"avg_ms": 0.923, "count": 128, "max_ms": 2.775, "p95_ms": 0.994}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.931, "count": 127, "max_ms": 72.279, "p95_ms": 61.073}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.967, "count": 127, "max_ms": 91.532, "p95_ms": 84.681}, "yolo11_640_fire_11class": {"avg_ms": 52.513, "count": 127, "max_ms": 71.311, "p95_ms": 66.143}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.101, "count": 127, "max_ms": 90.891, "p95_ms": 86.339}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.141, "count": 128, "max_ms": 308.744, "p95_ms": 254.808}, "read_interval": {"avg_ms": 234.824, "count": 128, "max_ms": 318.961, "p95_ms": 264.069}, "triton_wait": {"avg_ms": 67.343, "count": 127, "max_ms": 91.545, "p95_ms": 86.36}, "visualize": {"avg_ms": 10.509, "count": 127, "max_ms": 13.094, "p95_ms": 11.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 627}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4057.066, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T19:58:18+09:00", "last_error_summary": "[h264 @ 0x55abfbf33580] error while decoding MB 79 38, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 135, "h264_decode_error_count": 140, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:58:18+09:00", "last_error_summary": "[h264 @ 0x55abfbf33580] error while decoding MB 79 38, bytestream -5", "last_exit_at": "2026-07-01T19:58:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.606, "last_started_at": "2026-07-01T19:56:30+09:00", "pid": 20609, "process": {"exists": false, "pid": 20609}, "restart_count": 158, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 159}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 262.78, "sample_count": 1216, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75280, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:58:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T19:58:19+09:00", "last_frame_read_age_sec": 0.224, "last_frame_read_at": "2026-07-01T19:58:18+09:00", "mjpeg": {"age_sec": 0.214, "exists": true, "mtime": "2026-07-01T19:58:18.895328+09:00", "mtime_epoch": 1782903498.895328, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178147}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:58:19+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.148, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.294, "pushed": 4.294, "read": 4.26}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.789, "count": 128, "max_ms": 12.671, "p95_ms": 9.771}, "gstreamer_push": {"avg_ms": 0.923, "count": 128, "max_ms": 2.775, "p95_ms": 0.994}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.894, "count": 128, "max_ms": 72.279, "p95_ms": 61.073}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.911, "count": 128, "max_ms": 91.532, "p95_ms": 84.681}, "yolo11_640_fire_11class": {"avg_ms": 52.468, "count": 128, "max_ms": 71.311, "p95_ms": 66.143}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.051, "count": 128, "max_ms": 90.891, "p95_ms": 86.339}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.018, "count": 128, "max_ms": 308.744, "p95_ms": 254.808}, "read_interval": {"avg_ms": 234.736, "count": 127, "max_ms": 318.961, "p95_ms": 264.069}, "triton_wait": {"avg_ms": 67.283, "count": 128, "max_ms": 91.545, "p95_ms": 86.36}, "visualize": {"avg_ms": 10.512, "count": 128, "max_ms": 13.094, "p95_ms": 11.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.812, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 627}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4056.98, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 135, "h264_decode_error_count": 140, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:58:18+09:00", "last_error_summary": "[h264 @ 0x55abfbf33580] error while decoding MB 79 38, bytestream -5", "last_exit_at": "2026-07-01T19:58:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.609, "last_started_at": "2026-07-01T19:56:30+09:00", "pid": 20609, "process": {"exists": false, "pid": 20609}, "restart_count": 158, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 159}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 262.78, "sample_count": 1216, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75280, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:58:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T19:58:19+09:00", "last_frame_read_age_sec": 0.227, "last_frame_read_at": "2026-07-01T19:58:18+09:00", "mjpeg": {"age_sec": 0.217, "exists": true, "mtime": "2026-07-01T19:58:18.895328+09:00", "mtime_epoch": 1782903498.895328, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178147}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:58:19+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.148, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.293, "pushed": 4.293, "read": 4.26}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.789, "count": 128, "max_ms": 12.671, "p95_ms": 9.771}, "gstreamer_push": {"avg_ms": 0.923, "count": 128, "max_ms": 2.775, "p95_ms": 0.994}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.894, "count": 128, "max_ms": 72.279, "p95_ms": 61.073}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.911, "count": 128, "max_ms": 91.532, "p95_ms": 84.681}, "yolo11_640_fire_11class": {"avg_ms": 52.468, "count": 128, "max_ms": 71.311, "p95_ms": 66.143}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.051, "count": 128, "max_ms": 90.891, "p95_ms": 86.339}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.018, "count": 128, "max_ms": 308.744, "p95_ms": 254.808}, "read_interval": {"avg_ms": 234.736, "count": 127, "max_ms": 318.961, "p95_ms": 264.069}, "triton_wait": {"avg_ms": 67.283, "count": 128, "max_ms": 91.545, "p95_ms": 86.36}, "visualize": {"avg_ms": 10.512, "count": 128, "max_ms": 13.094, "p95_ms": 11.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.815, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 627}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4162.383, "limit_mb": 8796093022207.996}, "detail": {"pid": 20716, "restart_count": 159, "start_count": 160}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 135, "h264_decode_error_count": 140, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T19:58:18+09:00", "last_error_summary": "[h264 @ 0x55abfbf33580] error while decoding MB 79 38, bytestream -5", "last_exit_at": "2026-07-01T19:58:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T19:58:21+09:00", "pid": 20716, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.605, "name": "ffmpeg", "pid": 20716, "rss_mb": 138.605, "state": "S (sleeping)", "threads": 106}, "restart_count": 159, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 160}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 262.78, "sample_count": 1198, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75280, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T19:58:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.13, "last_frame_pushed_at": "2026-07-01T19:58:19+09:00", "last_frame_read_age_sec": 4.351, "last_frame_read_at": "2026-07-01T19:58:18+09:00", "mjpeg": {"age_sec": 4.113, "exists": true, "mtime": "2026-07-01T19:58:19.123332+09:00", "mtime_epoch": 1782903499.1233315, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 178049}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T19:58:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.148, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.676, "pushed": 3.676, "read": 3.642}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.884, "count": 109, "max_ms": 12.671, "p95_ms": 9.808}, "gstreamer_push": {"avg_ms": 0.923, "count": 110, "max_ms": 2.775, "p95_ms": 0.994}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.456, "count": 109, "max_ms": 68.863, "p95_ms": 59.927}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.85, "count": 109, "max_ms": 91.532, "p95_ms": 84.926}, "yolo11_640_fire_11class": {"avg_ms": 52.29, "count": 109, "max_ms": 71.311, "p95_ms": 66.143}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.007, "count": 109, "max_ms": 90.891, "p95_ms": 86.339}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.759, "count": 110, "max_ms": 308.744, "p95_ms": 256.119}, "read_interval": {"avg_ms": 236.506, "count": 109, "max_ms": 318.961, "p95_ms": 265.398}, "triton_wait": {"avg_ms": 67.196, "count": 109, "max_ms": 91.545, "p95_ms": 86.36}, "visualize": {"avg_ms": 10.589, "count": 109, "max_ms": 12.692, "p95_ms": 11.623}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 631}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4056.406, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56528e101580] error while decoding MB 70 40, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 136, "h264_decode_error_count": 141, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:00:11+09:00", "last_error_summary": "[h264 @ 0x56528e101580] error while decoding MB 70 40, bytestream -7", "last_exit_at": "2026-07-01T19:58:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.959, "last_started_at": "2026-07-01T19:58:21+09:00", "pid": 20716, "process": {"exists": false, "pid": 20716}, "restart_count": 159, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 160}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 254.286, "sample_count": 1230, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75740, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:00:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T20:00:11+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T20:00:11+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T20:00:11.149056+09:00", "mtime_epoch": 1782903611.149056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179602}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:00:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.238, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.301, "pushed": 4.301, "read": 4.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.801, "count": 128, "max_ms": 13.149, "p95_ms": 9.675}, "gstreamer_push": {"avg_ms": 0.992, "count": 129, "max_ms": 3.148, "p95_ms": 1.866}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.355, "count": 128, "max_ms": 67.053, "p95_ms": 61.666}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.297, "count": 128, "max_ms": 91.03, "p95_ms": 86.484}, "yolo11_640_fire_11class": {"avg_ms": 52.824, "count": 128, "max_ms": 70.374, "p95_ms": 67.439}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.035, "count": 128, "max_ms": 91.963, "p95_ms": 87.438}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.102, "count": 129, "max_ms": 314.991, "p95_ms": 257.552}, "read_interval": {"avg_ms": 233.813, "count": 129, "max_ms": 325.22, "p95_ms": 266.637}, "triton_wait": {"avg_ms": 67.478, "count": 128, "max_ms": 91.974, "p95_ms": 87.45}, "visualize": {"avg_ms": 10.412, "count": 128, "max_ms": 11.901, "p95_ms": 11.407}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 631}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4055.637, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:00:11+09:00", "last_error_summary": "[h264 @ 0x56528e101580] error while decoding MB 70 40, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 136, "h264_decode_error_count": 141, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:00:11+09:00", "last_error_summary": "[h264 @ 0x56528e101580] error while decoding MB 70 40, bytestream -7", "last_exit_at": "2026-07-01T20:00:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.134, "last_started_at": "2026-07-01T19:58:21+09:00", "pid": 20716, "process": {"exists": false, "pid": 20716}, "restart_count": 159, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 160}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 254.286, "sample_count": 1230, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75741, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:00:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T20:00:11+09:00", "last_frame_read_age_sec": 0.228, "last_frame_read_at": "2026-07-01T20:00:11+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-01T20:00:11.149056+09:00", "mtime_epoch": 1782903611.149056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179602}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:00:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.238, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.31, "pushed": 4.31, "read": 4.277}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.807, "count": 128, "max_ms": 13.149, "p95_ms": 9.675}, "gstreamer_push": {"avg_ms": 0.985, "count": 129, "max_ms": 3.148, "p95_ms": 1.828}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.37, "count": 128, "max_ms": 67.053, "p95_ms": 61.666}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.313, "count": 128, "max_ms": 91.03, "p95_ms": 86.484}, "yolo11_640_fire_11class": {"avg_ms": 52.835, "count": 128, "max_ms": 70.374, "p95_ms": 67.439}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.052, "count": 128, "max_ms": 91.963, "p95_ms": 87.438}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.094, "count": 129, "max_ms": 314.991, "p95_ms": 257.552}, "read_interval": {"avg_ms": 233.804, "count": 128, "max_ms": 325.22, "p95_ms": 266.637}, "triton_wait": {"avg_ms": 67.494, "count": 128, "max_ms": 91.974, "p95_ms": 87.45}, "visualize": {"avg_ms": 10.41, "count": 128, "max_ms": 11.901, "p95_ms": 11.407}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 631}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4055.629, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 136, "h264_decode_error_count": 141, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:00:11+09:00", "last_error_summary": "[h264 @ 0x56528e101580] error while decoding MB 70 40, bytestream -7", "last_exit_at": "2026-07-01T20:00:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 110.137, "last_started_at": "2026-07-01T19:58:21+09:00", "pid": 20716, "process": {"exists": false, "pid": 20716}, "restart_count": 159, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 160}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 254.286, "sample_count": 1230, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75741, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:00:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:00:11+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-01T20:00:11+09:00", "mjpeg": {"age_sec": 0.223, "exists": true, "mtime": "2026-07-01T20:00:11.149056+09:00", "mtime_epoch": 1782903611.149056, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179602}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:00:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.238, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.31, "pushed": 4.31, "read": 4.276}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.807, "count": 128, "max_ms": 13.149, "p95_ms": 9.675}, "gstreamer_push": {"avg_ms": 0.985, "count": 129, "max_ms": 3.148, "p95_ms": 1.828}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.37, "count": 128, "max_ms": 67.053, "p95_ms": 61.666}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.313, "count": 128, "max_ms": 91.03, "p95_ms": 86.484}, "yolo11_640_fire_11class": {"avg_ms": 52.835, "count": 128, "max_ms": 70.374, "p95_ms": 67.439}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.052, "count": 128, "max_ms": 91.963, "p95_ms": 87.438}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.094, "count": 129, "max_ms": 314.991, "p95_ms": 257.552}, "read_interval": {"avg_ms": 233.804, "count": 128, "max_ms": 325.22, "p95_ms": 266.637}, "triton_wait": {"avg_ms": 67.494, "count": 128, "max_ms": 91.974, "p95_ms": 87.45}, "visualize": {"avg_ms": 10.41, "count": 128, "max_ms": 11.901, "p95_ms": 11.407}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 631}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4166.191, "limit_mb": 8796093022207.996}, "detail": {"pid": 20823, "restart_count": 160, "start_count": 161}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 136, "h264_decode_error_count": 141, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:00:11+09:00", "last_error_summary": "[h264 @ 0x56528e101580] error while decoding MB 70 40, bytestream -7", "last_exit_at": "2026-07-01T20:00:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:00:14+09:00", "pid": 20823, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.172, "name": "ffmpeg", "pid": 20823, "rss_mb": 139.172, "state": "S (sleeping)", "threads": 106}, "restart_count": 160, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 161}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 254.286, "sample_count": 1210, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75741, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:00:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.916, "last_frame_pushed_at": "2026-07-01T20:00:11+09:00", "last_frame_read_age_sec": 5.141, "last_frame_read_at": "2026-07-01T20:00:11+09:00", "mjpeg": {"age_sec": 4.901, "exists": true, "mtime": "2026-07-01T20:00:11.381060+09:00", "mtime_epoch": 1782903611.3810596, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179551}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:00:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.238, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.575, "pushed": 3.575, "read": 3.541}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.936, "count": 106, "max_ms": 13.149, "p95_ms": 9.675}, "gstreamer_push": {"avg_ms": 0.996, "count": 107, "max_ms": 3.148, "p95_ms": 1.866}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.289, "count": 106, "max_ms": 67.053, "p95_ms": 61.326}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.758, "count": 106, "max_ms": 91.03, "p95_ms": 86.278}, "yolo11_640_fire_11class": {"avg_ms": 52.37, "count": 106, "max_ms": 70.374, "p95_ms": 66.493}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.472, "count": 106, "max_ms": 91.963, "p95_ms": 85.355}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.912, "count": 107, "max_ms": 314.991, "p95_ms": 257.552}, "read_interval": {"avg_ms": 235.653, "count": 106, "max_ms": 325.22, "p95_ms": 266.637}, "triton_wait": {"avg_ms": 66.907, "count": 106, "max_ms": 91.974, "p95_ms": 86.289}, "visualize": {"avg_ms": 10.56, "count": 106, "max_ms": 11.901, "p95_ms": 11.438}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 635}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4062.137, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557c688f1300] error while decoding MB 4 30, bytestream -17", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 137, "h264_decode_error_count": 142, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:02:02+09:00", "last_error_summary": "[h264 @ 0x557c688f1300] error while decoding MB 4 30, bytestream -17", "last_exit_at": "2026-07-01T20:00:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.646, "last_started_at": "2026-07-01T20:00:14+09:00", "pid": 20823, "process": {"exists": false, "pid": 20823}, "restart_count": 160, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 161}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.533, "sample_count": 1247, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:02:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T20:02:02+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T20:02:02+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T20:02:02.886775+09:00", "mtime_epoch": 1782903722.886775, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181105}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:02:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.512, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.39, "pushed": 4.39, "read": 4.39}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.362, "count": 131, "max_ms": 13.936, "p95_ms": 9.163}, "gstreamer_push": {"avg_ms": 0.937, "count": 131, "max_ms": 2.875, "p95_ms": 1.802}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.217, "count": 131, "max_ms": 69.983, "p95_ms": 64.954}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.833, "count": 131, "max_ms": 96.447, "p95_ms": 87.425}, "yolo11_640_fire_11class": {"avg_ms": 54.705, "count": 131, "max_ms": 69.463, "p95_ms": 68.123}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.803, "count": 131, "max_ms": 94.8, "p95_ms": 88.347}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.309, "count": 131, "max_ms": 306.024, "p95_ms": 241.968}, "read_interval": {"avg_ms": 229.043, "count": 131, "max_ms": 317.734, "p95_ms": 251.304}, "triton_wait": {"avg_ms": 70.223, "count": 131, "max_ms": 96.459, "p95_ms": 88.41}, "visualize": {"avg_ms": 10.139, "count": 131, "max_ms": 13.492, "p95_ms": 11.633}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.84, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 635}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4061.242, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:02:02+09:00", "last_error_summary": "[h264 @ 0x557c688f1300] error while decoding MB 4 30, bytestream -17", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 137, "h264_decode_error_count": 142, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:02:02+09:00", "last_error_summary": "[h264 @ 0x557c688f1300] error while decoding MB 4 30, bytestream -17", "last_exit_at": "2026-07-01T20:02:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.814, "last_started_at": "2026-07-01T20:00:14+09:00", "pid": 20823, "process": {"exists": false, "pid": 20823}, "restart_count": 160, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 161}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.533, "sample_count": 1247, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76206, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:02:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T20:02:03+09:00", "last_frame_read_age_sec": 0.219, "last_frame_read_at": "2026-07-01T20:02:02+09:00", "mjpeg": {"age_sec": 0.208, "exists": true, "mtime": "2026-07-01T20:02:02.886775+09:00", "mtime_epoch": 1782903722.886775, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181105}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:02:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.512, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.367, "pushed": 4.367, "read": 4.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.351, "count": 131, "max_ms": 13.936, "p95_ms": 9.163}, "gstreamer_push": {"avg_ms": 0.937, "count": 131, "max_ms": 2.875, "p95_ms": 1.802}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.219, "count": 131, "max_ms": 69.983, "p95_ms": 64.954}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.814, "count": 131, "max_ms": 96.447, "p95_ms": 87.425}, "yolo11_640_fire_11class": {"avg_ms": 54.732, "count": 131, "max_ms": 69.463, "p95_ms": 68.123}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.811, "count": 131, "max_ms": 94.8, "p95_ms": 88.347}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.292, "count": 131, "max_ms": 306.024, "p95_ms": 241.968}, "read_interval": {"avg_ms": 229.043, "count": 131, "max_ms": 317.734, "p95_ms": 251.304}, "triton_wait": {"avg_ms": 70.204, "count": 131, "max_ms": 96.459, "p95_ms": 88.41}, "visualize": {"avg_ms": 10.138, "count": 131, "max_ms": 13.492, "p95_ms": 11.633}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 635}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4061.172, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 137, "h264_decode_error_count": 142, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:02:02+09:00", "last_error_summary": "[h264 @ 0x557c688f1300] error while decoding MB 4 30, bytestream -17", "last_exit_at": "2026-07-01T20:02:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.817, "last_started_at": "2026-07-01T20:00:14+09:00", "pid": 20823, "process": {"exists": false, "pid": 20823}, "restart_count": 160, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 161}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.533, "sample_count": 1247, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76206, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:02:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:02:03+09:00", "last_frame_read_age_sec": 0.222, "last_frame_read_at": "2026-07-01T20:02:02+09:00", "mjpeg": {"age_sec": 0.211, "exists": true, "mtime": "2026-07-01T20:02:02.886775+09:00", "mtime_epoch": 1782903722.886775, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181105}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:02:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.512, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.398, "pushed": 4.398, "read": 4.364}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.351, "count": 131, "max_ms": 13.936, "p95_ms": 9.163}, "gstreamer_push": {"avg_ms": 0.937, "count": 131, "max_ms": 2.875, "p95_ms": 1.802}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.219, "count": 131, "max_ms": 69.983, "p95_ms": 64.954}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.814, "count": 131, "max_ms": 96.447, "p95_ms": 87.425}, "yolo11_640_fire_11class": {"avg_ms": 54.732, "count": 131, "max_ms": 69.463, "p95_ms": 68.123}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.811, "count": 131, "max_ms": 94.8, "p95_ms": 88.347}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.292, "count": 131, "max_ms": 306.024, "p95_ms": 241.968}, "read_interval": {"avg_ms": 229.063, "count": 130, "max_ms": 317.734, "p95_ms": 251.304}, "triton_wait": {"avg_ms": 70.204, "count": 131, "max_ms": 96.459, "p95_ms": 88.41}, "visualize": {"avg_ms": 10.138, "count": 131, "max_ms": 13.492, "p95_ms": 11.633}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.786, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 635}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4172.18, "limit_mb": 8796093022207.996}, "detail": {"pid": 20930, "restart_count": 161, "start_count": 162}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 137, "h264_decode_error_count": 142, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:02:02+09:00", "last_error_summary": "[h264 @ 0x557c688f1300] error while decoding MB 4 30, bytestream -17", "last_exit_at": "2026-07-01T20:02:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:02:05+09:00", "pid": 20930, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.777, "name": "ffmpeg", "pid": 20930, "rss_mb": 144.777, "state": "S (sleeping)", "threads": 106}, "restart_count": 161, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 162}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.787, "sample_count": 1228, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76206, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:02:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.626, "last_frame_pushed_at": "2026-07-01T20:02:03+09:00", "last_frame_read_age_sec": 4.842, "last_frame_read_at": "2026-07-01T20:02:02+09:00", "mjpeg": {"age_sec": 4.607, "exists": true, "mtime": "2026-07-01T20:02:03.110778+09:00", "mtime_epoch": 1782903723.1107783, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181173}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:02:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.512, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.717, "pushed": 3.717, "read": 3.684}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.347, "count": 111, "max_ms": 13.936, "p95_ms": 9.163}, "gstreamer_push": {"avg_ms": 0.95, "count": 111, "max_ms": 2.875, "p95_ms": 1.845}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.822, "count": 111, "max_ms": 68.4, "p95_ms": 64.954}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.894, "count": 111, "max_ms": 96.447, "p95_ms": 87.425}, "yolo11_640_fire_11class": {"avg_ms": 54.596, "count": 111, "max_ms": 68.987, "p95_ms": 68.123}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.95, "count": 111, "max_ms": 94.8, "p95_ms": 88.347}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.57, "count": 111, "max_ms": 306.024, "p95_ms": 248.19}, "read_interval": {"avg_ms": 229.364, "count": 110, "max_ms": 317.734, "p95_ms": 257.463}, "triton_wait": {"avg_ms": 70.323, "count": 111, "max_ms": 96.459, "p95_ms": 88.41}, "visualize": {"avg_ms": 10.142, "count": 111, "max_ms": 13.492, "p95_ms": 12.0}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.861, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 638}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4060.496, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d8f9e7d1c0] error while decoding MB 33 10, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 138, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:03:55+09:00", "last_error_summary": "[h264 @ 0x55d8f9e7d1c0] error while decoding MB 33 10, bytestream -6", "last_exit_at": "2026-07-01T20:02:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.327, "last_started_at": "2026-07-01T20:02:05+09:00", "pid": 20930, "process": {"exists": false, "pid": 20930}, "restart_count": 161, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 162}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 251.322, "sample_count": 1250, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76664, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:03:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T20:03:54+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T20:03:54+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T20:03:55.002264+09:00", "mtime_epoch": 1782903835.0022638, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182897}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:03:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 895.621, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.226, "pushed": 4.226, "read": 4.226}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.874, "count": 126, "max_ms": 10.126, "p95_ms": 9.424}, "gstreamer_push": {"avg_ms": 0.896, "count": 126, "max_ms": 3.062, "p95_ms": 0.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.182, "count": 126, "max_ms": 67.28, "p95_ms": 59.149}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.479, "count": 126, "max_ms": 99.409, "p95_ms": 82.629}, "yolo11_640_fire_11class": {"avg_ms": 53.002, "count": 126, "max_ms": 71.98, "p95_ms": 66.579}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.908, "count": 126, "max_ms": 98.202, "p95_ms": 84.651}}, "truncated_count": 0}, "processing_total": {"avg_ms": 228.618, "count": 126, "max_ms": 274.83, "p95_ms": 252.236}, "read_interval": {"avg_ms": 238.012, "count": 126, "max_ms": 285.54, "p95_ms": 261.22}, "triton_wait": {"avg_ms": 67.813, "count": 126, "max_ms": 99.423, "p95_ms": 85.531}, "visualize": {"avg_ms": 10.678, "count": 126, "max_ms": 12.272, "p95_ms": 11.351}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.817, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 638}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4059.68, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:03:55+09:00", "last_error_summary": "[h264 @ 0x55d8f9e7d1c0] error while decoding MB 33 10, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 138, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:03:55+09:00", "last_error_summary": "[h264 @ 0x55d8f9e7d1c0] error while decoding MB 33 10, bytestream -6", "last_exit_at": "2026-07-01T20:03:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.517, "last_started_at": "2026-07-01T20:02:05+09:00", "pid": 20930, "process": {"exists": false, "pid": 20930}, "restart_count": 161, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 162}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 251.322, "sample_count": 1251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76665, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:03:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T20:03:55+09:00", "last_frame_read_age_sec": 0.24, "last_frame_read_at": "2026-07-01T20:03:54+09:00", "mjpeg": {"age_sec": 0.231, "exists": true, "mtime": "2026-07-01T20:03:55.002264+09:00", "mtime_epoch": 1782903835.0022638, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182897}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:03:55+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 895.621, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.2, "pushed": 4.2, "read": 4.2}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.874, "count": 126, "max_ms": 10.126, "p95_ms": 9.424}, "gstreamer_push": {"avg_ms": 0.896, "count": 126, "max_ms": 3.062, "p95_ms": 0.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.261, "count": 126, "max_ms": 67.28, "p95_ms": 59.149}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.54, "count": 126, "max_ms": 99.409, "p95_ms": 82.629}, "yolo11_640_fire_11class": {"avg_ms": 53.094, "count": 126, "max_ms": 71.98, "p95_ms": 66.579}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.993, "count": 126, "max_ms": 98.202, "p95_ms": 84.651}}, "truncated_count": 0}, "processing_total": {"avg_ms": 228.714, "count": 126, "max_ms": 274.83, "p95_ms": 252.236}, "read_interval": {"avg_ms": 238.012, "count": 126, "max_ms": 285.54, "p95_ms": 261.22}, "triton_wait": {"avg_ms": 67.891, "count": 126, "max_ms": 99.423, "p95_ms": 85.531}, "visualize": {"avg_ms": 10.679, "count": 126, "max_ms": 12.272, "p95_ms": 11.351}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 638}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4059.672, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 138, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:03:55+09:00", "last_error_summary": "[h264 @ 0x55d8f9e7d1c0] error while decoding MB 33 10, bytestream -6", "last_exit_at": "2026-07-01T20:03:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.52, "last_started_at": "2026-07-01T20:02:05+09:00", "pid": 20930, "process": {"exists": false, "pid": 20930}, "restart_count": 161, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 162}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 251.322, "sample_count": 1251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76665, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:03:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:03:55+09:00", "last_frame_read_age_sec": 0.243, "last_frame_read_at": "2026-07-01T20:03:54+09:00", "mjpeg": {"age_sec": 0.234, "exists": true, "mtime": "2026-07-01T20:03:55.002264+09:00", "mtime_epoch": 1782903835.0022638, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182897}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:03:55+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 895.621, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.232, "pushed": 4.232, "read": 4.199}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.874, "count": 126, "max_ms": 10.126, "p95_ms": 9.424}, "gstreamer_push": {"avg_ms": 0.896, "count": 126, "max_ms": 3.062, "p95_ms": 0.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.261, "count": 126, "max_ms": 67.28, "p95_ms": 59.149}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.54, "count": 126, "max_ms": 99.409, "p95_ms": 82.629}, "yolo11_640_fire_11class": {"avg_ms": 53.094, "count": 126, "max_ms": 71.98, "p95_ms": 66.579}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.993, "count": 126, "max_ms": 98.202, "p95_ms": 84.651}}, "truncated_count": 0}, "processing_total": {"avg_ms": 228.714, "count": 126, "max_ms": 274.83, "p95_ms": 252.236}, "read_interval": {"avg_ms": 238.056, "count": 125, "max_ms": 285.54, "p95_ms": 261.22}, "triton_wait": {"avg_ms": 67.891, "count": 126, "max_ms": 99.423, "p95_ms": 85.531}, "visualize": {"avg_ms": 10.679, "count": 126, "max_ms": 12.272, "p95_ms": 11.351}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.772, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 638}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4168.277, "limit_mb": 8796093022207.996}, "detail": {"pid": 21037, "restart_count": 162, "start_count": 163}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 138, "h264_decode_error_count": 143, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:03:55+09:00", "last_error_summary": "[h264 @ 0x55d8f9e7d1c0] error while decoding MB 33 10, bytestream -6", "last_exit_at": "2026-07-01T20:03:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:03:57+09:00", "pid": 21037, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.215, "name": "ffmpeg", "pid": 21037, "rss_mb": 139.215, "state": "S (sleeping)", "threads": 106}, "restart_count": 162, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 163}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.893, "sample_count": 1232, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76665, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:03:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.141, "last_frame_pushed_at": "2026-07-01T20:03:55+09:00", "last_frame_read_age_sec": 4.378, "last_frame_read_at": "2026-07-01T20:03:54+09:00", "mjpeg": {"age_sec": 4.125, "exists": true, "mtime": "2026-07-01T20:03:55.246272+09:00", "mtime_epoch": 1782903835.246272, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182948}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:03:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 895.621, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.644, "pushed": 3.644, "read": 3.61}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.833, "count": 108, "max_ms": 10.126, "p95_ms": 9.604}, "gstreamer_push": {"avg_ms": 0.904, "count": 109, "max_ms": 3.062, "p95_ms": 0.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.948, "count": 108, "max_ms": 67.28, "p95_ms": 59.149}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.704, "count": 108, "max_ms": 99.409, "p95_ms": 83.938}, "yolo11_640_fire_11class": {"avg_ms": 53.125, "count": 108, "max_ms": 71.98, "p95_ms": 67.566}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.104, "count": 108, "max_ms": 98.202, "p95_ms": 84.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.244, "count": 109, "max_ms": 274.83, "p95_ms": 253.141}, "read_interval": {"avg_ms": 238.611, "count": 108, "max_ms": 285.54, "p95_ms": 262.257}, "triton_wait": {"avg_ms": 68.04, "count": 108, "max_ms": 99.423, "p95_ms": 86.084}, "visualize": {"avg_ms": 10.696, "count": 108, "max_ms": 12.272, "p95_ms": 11.367}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 642}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4061.137, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a215ec12c0] error while decoding MB 13 1, bytestream -17", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 139, "h264_decode_error_count": 144, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:05:46+09:00", "last_error_summary": "[h264 @ 0x55a215ec12c0] error while decoding MB 13 1, bytestream -17", "last_exit_at": "2026-07-01T20:03:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.173, "last_started_at": "2026-07-01T20:03:57+09:00", "pid": 21037, "process": {"exists": false, "pid": 21037}, "restart_count": 162, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 163}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.533, "sample_count": 1249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77124, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:05:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T20:05:46+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T20:05:46+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T20:05:46.501859+09:00", "mtime_epoch": 1782903946.5018594, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182909}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:05:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.031, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.319, "pushed": 4.319, "read": 4.319}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.6, "count": 128, "max_ms": 13.258, "p95_ms": 9.893}, "gstreamer_push": {"avg_ms": 0.934, "count": 129, "max_ms": 2.917, "p95_ms": 1.091}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.285, "count": 128, "max_ms": 70.046, "p95_ms": 61.989}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.174, "count": 128, "max_ms": 90.301, "p95_ms": 88.058}, "yolo11_640_fire_11class": {"avg_ms": 53.571, "count": 128, "max_ms": 70.278, "p95_ms": 68.229}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.977, "count": 128, "max_ms": 90.388, "p95_ms": 89.007}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.28, "count": 129, "max_ms": 300.374, "p95_ms": 255.592}, "read_interval": {"avg_ms": 232.878, "count": 129, "max_ms": 309.735, "p95_ms": 267.339}, "triton_wait": {"avg_ms": 69.536, "count": 128, "max_ms": 90.401, "p95_ms": 89.331}, "visualize": {"avg_ms": 10.319, "count": 128, "max_ms": 13.604, "p95_ms": 12.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.865, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 642}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4060.477, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:05:46+09:00", "last_error_summary": "[h264 @ 0x55a215ec12c0] error while decoding MB 13 1, bytestream -17", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 139, "h264_decode_error_count": 144, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:05:46+09:00", "last_error_summary": "[h264 @ 0x55a215ec12c0] error while decoding MB 13 1, bytestream -17", "last_exit_at": "2026-07-01T20:05:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.363, "last_started_at": "2026-07-01T20:03:57+09:00", "pid": 21037, "process": {"exists": false, "pid": 21037}, "restart_count": 162, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 163}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.533, "sample_count": 1249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77125, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:05:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T20:05:46+09:00", "last_frame_read_age_sec": 0.24, "last_frame_read_at": "2026-07-01T20:05:46+09:00", "mjpeg": {"age_sec": 0.232, "exists": true, "mtime": "2026-07-01T20:05:46.501859+09:00", "mtime_epoch": 1782903946.5018594, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182909}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:05:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.031, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.327, "pushed": 4.327, "read": 4.294}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.603, "count": 129, "max_ms": 13.258, "p95_ms": 9.893}, "gstreamer_push": {"avg_ms": 0.934, "count": 129, "max_ms": 2.917, "p95_ms": 1.091}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.218, "count": 129, "max_ms": 70.046, "p95_ms": 61.989}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.128, "count": 129, "max_ms": 90.301, "p95_ms": 88.058}, "yolo11_640_fire_11class": {"avg_ms": 53.512, "count": 129, "max_ms": 70.278, "p95_ms": 68.229}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.902, "count": 129, "max_ms": 90.388, "p95_ms": 89.007}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.348, "count": 129, "max_ms": 300.374, "p95_ms": 255.592}, "read_interval": {"avg_ms": 232.855, "count": 128, "max_ms": 309.735, "p95_ms": 267.339}, "triton_wait": {"avg_ms": 69.488, "count": 129, "max_ms": 90.401, "p95_ms": 89.331}, "visualize": {"avg_ms": 10.327, "count": 129, "max_ms": 13.604, "p95_ms": 12.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.812, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 642}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4060.469, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 139, "h264_decode_error_count": 144, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:05:46+09:00", "last_error_summary": "[h264 @ 0x55a215ec12c0] error while decoding MB 13 1, bytestream -17", "last_exit_at": "2026-07-01T20:05:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.366, "last_started_at": "2026-07-01T20:03:57+09:00", "pid": 21037, "process": {"exists": false, "pid": 21037}, "restart_count": 162, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 163}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.533, "sample_count": 1249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77125, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:05:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:05:46+09:00", "last_frame_read_age_sec": 0.243, "last_frame_read_at": "2026-07-01T20:05:46+09:00", "mjpeg": {"age_sec": 0.235, "exists": true, "mtime": "2026-07-01T20:05:46.501859+09:00", "mtime_epoch": 1782903946.5018594, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182909}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:05:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.031, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.327, "pushed": 4.327, "read": 4.293}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.603, "count": 129, "max_ms": 13.258, "p95_ms": 9.893}, "gstreamer_push": {"avg_ms": 0.934, "count": 129, "max_ms": 2.917, "p95_ms": 1.091}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.218, "count": 129, "max_ms": 70.046, "p95_ms": 61.989}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.128, "count": 129, "max_ms": 90.301, "p95_ms": 88.058}, "yolo11_640_fire_11class": {"avg_ms": 53.512, "count": 129, "max_ms": 70.278, "p95_ms": 68.229}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.902, "count": 129, "max_ms": 90.388, "p95_ms": 89.007}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.348, "count": 129, "max_ms": 300.374, "p95_ms": 255.592}, "read_interval": {"avg_ms": 232.855, "count": 128, "max_ms": 309.735, "p95_ms": 267.339}, "triton_wait": {"avg_ms": 69.488, "count": 129, "max_ms": 90.401, "p95_ms": 89.331}, "visualize": {"avg_ms": 10.327, "count": 129, "max_ms": 13.604, "p95_ms": 12.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.815, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 642}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4168.215, "limit_mb": 8796093022207.996}, "detail": {"pid": 21144, "restart_count": 163, "start_count": 164}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 139, "h264_decode_error_count": 144, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:05:46+09:00", "last_error_summary": "[h264 @ 0x55a215ec12c0] error while decoding MB 13 1, bytestream -17", "last_exit_at": "2026-07-01T20:05:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:05:49+09:00", "pid": 21144, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.395, "name": "ffmpeg", "pid": 21144, "rss_mb": 139.395, "state": "S (sleeping)", "threads": 106}, "restart_count": 163, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 164}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.236, "sample_count": 1229, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77125, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:05:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.634, "last_frame_pushed_at": "2026-07-01T20:05:46+09:00", "last_frame_read_age_sec": 4.871, "last_frame_read_at": "2026-07-01T20:05:46+09:00", "mjpeg": {"age_sec": 4.615, "exists": true, "mtime": "2026-07-01T20:05:46.749867+09:00", "mtime_epoch": 1782903946.749867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183026}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:05:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.031, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.654, "pushed": 3.654, "read": 3.62}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.575, "count": 109, "max_ms": 13.258, "p95_ms": 10.021}, "gstreamer_push": {"avg_ms": 0.947, "count": 109, "max_ms": 2.917, "p95_ms": 1.819}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.234, "count": 109, "max_ms": 70.046, "p95_ms": 62.11}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.556, "count": 109, "max_ms": 90.301, "p95_ms": 88.058}, "yolo11_640_fire_11class": {"avg_ms": 53.576, "count": 109, "max_ms": 69.5, "p95_ms": 68.229}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.316, "count": 109, "max_ms": 90.388, "p95_ms": 89.007}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.773, "count": 109, "max_ms": 300.374, "p95_ms": 256.042}, "read_interval": {"avg_ms": 233.284, "count": 108, "max_ms": 309.735, "p95_ms": 267.914}, "triton_wait": {"avg_ms": 69.922, "count": 109, "max_ms": 90.401, "p95_ms": 89.331}, "visualize": {"avg_ms": 10.352, "count": 109, "max_ms": 13.604, "p95_ms": 12.642}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.831, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 647}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4062.371, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b8f4814180] error while decoding MB 11 23, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 140, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:08:01+09:00", "last_error_summary": "[h264 @ 0x55b8f4814180] error while decoding MB 11 23, bytestream -5", "last_exit_at": "2026-07-01T20:05:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.5, "last_started_at": "2026-07-01T20:05:49+09:00", "pid": 21144, "process": {"exists": false, "pid": 21144}, "restart_count": 163, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 164}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 251.343, "sample_count": 1249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77689, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:08:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-01T20:08:01+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T20:08:01+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-01T20:08:01.821832+09:00", "mtime_epoch": 1782904081.8218317, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183113}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:08:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 898.863, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.422, "pushed": 4.422, "read": 4.422}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.517, "count": 132, "max_ms": 13.244, "p95_ms": 10.495}, "gstreamer_push": {"avg_ms": 0.923, "count": 132, "max_ms": 1.88, "p95_ms": 1.786}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.132, "count": 132, "max_ms": 67.062, "p95_ms": 59.541}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.605, "count": 132, "max_ms": 89.204, "p95_ms": 86.837}, "yolo11_640_fire_11class": {"avg_ms": 52.536, "count": 132, "max_ms": 70.18, "p95_ms": 64.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.357, "count": 132, "max_ms": 88.251, "p95_ms": 86.516}}, "truncated_count": 0}, "processing_total": {"avg_ms": 217.829, "count": 132, "max_ms": 264.244, "p95_ms": 243.871}, "read_interval": {"avg_ms": 227.356, "count": 132, "max_ms": 274.487, "p95_ms": 253.521}, "triton_wait": {"avg_ms": 67.865, "count": 132, "max_ms": 89.216, "p95_ms": 86.847}, "visualize": {"avg_ms": 10.223, "count": 132, "max_ms": 14.147, "p95_ms": 11.894}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.852, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 647}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4061.762, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:08:01+09:00", "last_error_summary": "[h264 @ 0x55b8f4814180] error while decoding MB 11 23, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 140, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:08:01+09:00", "last_error_summary": "[h264 @ 0x55b8f4814180] error while decoding MB 11 23, bytestream -5", "last_exit_at": "2026-07-01T20:08:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.635, "last_started_at": "2026-07-01T20:05:49+09:00", "pid": 21144, "process": {"exists": false, "pid": 21144}, "restart_count": 163, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 164}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 251.343, "sample_count": 1249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77690, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:08:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T20:08:01+09:00", "last_frame_read_age_sec": 0.187, "last_frame_read_at": "2026-07-01T20:08:01+09:00", "mjpeg": {"age_sec": 0.178, "exists": true, "mtime": "2026-07-01T20:08:01.821832+09:00", "mtime_epoch": 1782904081.8218317, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183113}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:08:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 898.863, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.435, "pushed": 4.435, "read": 4.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.51, "count": 132, "max_ms": 13.244, "p95_ms": 10.495}, "gstreamer_push": {"avg_ms": 0.923, "count": 133, "max_ms": 1.88, "p95_ms": 1.786}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.047, "count": 132, "max_ms": 67.062, "p95_ms": 59.541}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.568, "count": 132, "max_ms": 89.204, "p95_ms": 86.837}, "yolo11_640_fire_11class": {"avg_ms": 52.504, "count": 132, "max_ms": 70.18, "p95_ms": 64.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.314, "count": 132, "max_ms": 88.251, "p95_ms": 86.516}}, "truncated_count": 0}, "processing_total": {"avg_ms": 217.566, "count": 133, "max_ms": 264.244, "p95_ms": 243.871}, "read_interval": {"avg_ms": 227.356, "count": 132, "max_ms": 274.487, "p95_ms": 253.521}, "triton_wait": {"avg_ms": 67.828, "count": 132, "max_ms": 89.216, "p95_ms": 86.847}, "visualize": {"avg_ms": 10.213, "count": 132, "max_ms": 14.147, "p95_ms": 11.894}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 647}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4061.754, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 140, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:08:01+09:00", "last_error_summary": "[h264 @ 0x55b8f4814180] error while decoding MB 11 23, bytestream -5", "last_exit_at": "2026-07-01T20:08:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.639, "last_started_at": "2026-07-01T20:05:49+09:00", "pid": 21144, "process": {"exists": false, "pid": 21144}, "restart_count": 163, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 164}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 251.343, "sample_count": 1249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77690, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:08:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:08:01+09:00", "last_frame_read_age_sec": 0.191, "last_frame_read_at": "2026-07-01T20:08:01+09:00", "mjpeg": {"age_sec": 0.181, "exists": true, "mtime": "2026-07-01T20:08:01.821832+09:00", "mtime_epoch": 1782904081.8218317, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183113}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:08:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 898.863, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.435, "pushed": 4.435, "read": 4.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.51, "count": 132, "max_ms": 13.244, "p95_ms": 10.495}, "gstreamer_push": {"avg_ms": 0.923, "count": 133, "max_ms": 1.88, "p95_ms": 1.786}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.047, "count": 132, "max_ms": 67.062, "p95_ms": 59.541}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.568, "count": 132, "max_ms": 89.204, "p95_ms": 86.837}, "yolo11_640_fire_11class": {"avg_ms": 52.504, "count": 132, "max_ms": 70.18, "p95_ms": 64.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.314, "count": 132, "max_ms": 88.251, "p95_ms": 86.516}}, "truncated_count": 0}, "processing_total": {"avg_ms": 217.566, "count": 133, "max_ms": 264.244, "p95_ms": 243.871}, "read_interval": {"avg_ms": 227.356, "count": 132, "max_ms": 274.487, "p95_ms": 253.521}, "triton_wait": {"avg_ms": 67.828, "count": 132, "max_ms": 89.216, "p95_ms": 86.847}, "visualize": {"avg_ms": 10.213, "count": 132, "max_ms": 14.147, "p95_ms": 11.894}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 647}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4170.02, "limit_mb": 8796093022207.996}, "detail": {"pid": 21251, "restart_count": 164, "start_count": 165}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 140, "h264_decode_error_count": 145, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:08:01+09:00", "last_error_summary": "[h264 @ 0x55b8f4814180] error while decoding MB 11 23, bytestream -5", "last_exit_at": "2026-07-01T20:08:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:08:04+09:00", "pid": 21251, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.262, "name": "ffmpeg", "pid": 21251, "rss_mb": 139.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 164, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 165}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 251.327, "sample_count": 1232, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77690, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:08:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.282, "last_frame_pushed_at": "2026-07-01T20:08:01+09:00", "last_frame_read_age_sec": 4.467, "last_frame_read_at": "2026-07-01T20:08:01+09:00", "mjpeg": {"age_sec": 4.265, "exists": true, "mtime": "2026-07-01T20:08:02.013837+09:00", "mtime_epoch": 1782904082.013837, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183529}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:08:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 898.863, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.806, "pushed": 3.806, "read": 3.773}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.495, "count": 113, "max_ms": 11.629, "p95_ms": 10.495}, "gstreamer_push": {"avg_ms": 0.913, "count": 114, "max_ms": 1.88, "p95_ms": 1.087}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.058, "count": 113, "max_ms": 67.062, "p95_ms": 59.541}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.361, "count": 113, "max_ms": 89.204, "p95_ms": 86.837}, "yolo11_640_fire_11class": {"avg_ms": 53.453, "count": 113, "max_ms": 68.832, "p95_ms": 64.261}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.947, "count": 113, "max_ms": 88.251, "p95_ms": 86.516}}, "truncated_count": 0}, "processing_total": {"avg_ms": 217.65, "count": 114, "max_ms": 264.244, "p95_ms": 243.138}, "read_interval": {"avg_ms": 227.433, "count": 113, "max_ms": 274.487, "p95_ms": 252.424}, "triton_wait": {"avg_ms": 68.62, "count": 113, "max_ms": 89.216, "p95_ms": 86.847}, "visualize": {"avg_ms": 10.175, "count": 113, "max_ms": 14.147, "p95_ms": 11.845}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4065.527, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5648497b5d80] error while decoding MB 63 10, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:09:53+09:00", "last_error_summary": "[h264 @ 0x5648497b5d80] error while decoding MB 63 10, bytestream -6", "last_exit_at": "2026-07-01T20:08:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.046, "last_started_at": "2026-07-01T20:08:04+09:00", "pid": 21251, "process": {"exists": false, "pid": 21251}, "restart_count": 164, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 165}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.629, "sample_count": 1254, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78150, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:09:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-01T20:09:53+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-01T20:09:53+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-01T20:09:53.284848+09:00", "mtime_epoch": 1782904193.2848482, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182895}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:09:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.988, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.374, "pushed": 4.374, "read": 4.374}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.474, "count": 130, "max_ms": 12.914, "p95_ms": 10.984}, "gstreamer_push": {"avg_ms": 0.93, "count": 131, "max_ms": 2.937, "p95_ms": 1.034}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.979, "count": 130, "max_ms": 70.418, "p95_ms": 62.438}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.534, "count": 130, "max_ms": 98.738, "p95_ms": 87.463}, "yolo11_640_fire_11class": {"avg_ms": 54.077, "count": 130, "max_ms": 75.546, "p95_ms": 66.851}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.175, "count": 130, "max_ms": 95.978, "p95_ms": 87.445}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.948, "count": 131, "max_ms": 295.208, "p95_ms": 249.442}, "read_interval": {"avg_ms": 229.787, "count": 131, "max_ms": 305.625, "p95_ms": 258.955}, "triton_wait": {"avg_ms": 69.746, "count": 130, "max_ms": 98.751, "p95_ms": 87.547}, "visualize": {"avg_ms": 10.518, "count": 130, "max_ms": 16.691, "p95_ms": 14.218}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4064.898, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:09:53+09:00", "last_error_summary": "[h264 @ 0x5648497b5d80] error while decoding MB 63 10, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:09:53+09:00", "last_error_summary": "[h264 @ 0x5648497b5d80] error while decoding MB 63 10, bytestream -6", "last_exit_at": "2026-07-01T20:09:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.201, "last_started_at": "2026-07-01T20:08:04+09:00", "pid": 21251, "process": {"exists": false, "pid": 21251}, "restart_count": 164, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 165}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.629, "sample_count": 1255, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78151, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:09:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T20:09:53+09:00", "last_frame_read_age_sec": 0.205, "last_frame_read_at": "2026-07-01T20:09:53+09:00", "mjpeg": {"age_sec": 0.194, "exists": true, "mtime": "2026-07-01T20:09:53.284848+09:00", "mtime_epoch": 1782904193.2848482, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182895}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:09:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.988, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.384, "pushed": 4.384, "read": 4.351}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.473, "count": 130, "max_ms": 12.914, "p95_ms": 10.984}, "gstreamer_push": {"avg_ms": 0.93, "count": 131, "max_ms": 2.937, "p95_ms": 1.034}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.882, "count": 130, "max_ms": 70.418, "p95_ms": 62.438}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.464, "count": 130, "max_ms": 98.738, "p95_ms": 87.463}, "yolo11_640_fire_11class": {"avg_ms": 53.994, "count": 130, "max_ms": 75.546, "p95_ms": 66.851}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.106, "count": 130, "max_ms": 95.978, "p95_ms": 87.445}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.918, "count": 131, "max_ms": 295.208, "p95_ms": 249.442}, "read_interval": {"avg_ms": 229.894, "count": 130, "max_ms": 305.625, "p95_ms": 258.955}, "triton_wait": {"avg_ms": 69.677, "count": 130, "max_ms": 98.751, "p95_ms": 87.547}, "visualize": {"avg_ms": 10.516, "count": 130, "max_ms": 16.691, "p95_ms": 14.218}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4065.0, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:09:53+09:00", "last_error_summary": "[h264 @ 0x5648497b5d80] error while decoding MB 63 10, bytestream -6", "last_exit_at": "2026-07-01T20:09:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.204, "last_started_at": "2026-07-01T20:08:04+09:00", "pid": 21251, "process": {"exists": false, "pid": 21251}, "restart_count": 164, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 165}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.629, "sample_count": 1255, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78151, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:09:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:09:53+09:00", "last_frame_read_age_sec": 0.209, "last_frame_read_at": "2026-07-01T20:09:53+09:00", "mjpeg": {"age_sec": 0.197, "exists": true, "mtime": "2026-07-01T20:09:53.284848+09:00", "mtime_epoch": 1782904193.2848482, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182895}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:09:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.988, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.384, "pushed": 4.384, "read": 4.35}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.473, "count": 130, "max_ms": 12.914, "p95_ms": 10.984}, "gstreamer_push": {"avg_ms": 0.93, "count": 131, "max_ms": 2.937, "p95_ms": 1.034}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.882, "count": 130, "max_ms": 70.418, "p95_ms": 62.438}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.464, "count": 130, "max_ms": 98.738, "p95_ms": 87.463}, "yolo11_640_fire_11class": {"avg_ms": 53.994, "count": 130, "max_ms": 75.546, "p95_ms": 66.851}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.106, "count": 130, "max_ms": 95.978, "p95_ms": 87.445}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.918, "count": 131, "max_ms": 295.208, "p95_ms": 249.442}, "read_interval": {"avg_ms": 229.894, "count": 130, "max_ms": 305.625, "p95_ms": 258.955}, "triton_wait": {"avg_ms": 69.677, "count": 130, "max_ms": 98.751, "p95_ms": 87.547}, "visualize": {"avg_ms": 10.516, "count": 130, "max_ms": 16.691, "p95_ms": 14.218}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 651}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4176.281, "limit_mb": 8796093022207.996}, "detail": {"pid": 21358, "restart_count": 165, "start_count": 166}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 141, "h264_decode_error_count": 146, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:09:53+09:00", "last_error_summary": "[h264 @ 0x5648497b5d80] error while decoding MB 63 10, bytestream -6", "last_exit_at": "2026-07-01T20:09:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:09:55+09:00", "pid": 21358, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.406, "name": "ffmpeg", "pid": 21358, "rss_mb": 141.406, "state": "S (sleeping)", "threads": 106}, "restart_count": 165, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 166}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.357, "sample_count": 1237, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78151, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:09:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.336, "last_frame_pushed_at": "2026-07-01T20:09:53+09:00", "last_frame_read_age_sec": 4.538, "last_frame_read_at": "2026-07-01T20:09:53+09:00", "mjpeg": {"age_sec": 4.319, "exists": true, "mtime": "2026-07-01T20:09:53.492854+09:00", "mtime_epoch": 1782904193.4928536, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182973}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:09:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 900.988, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.747, "pushed": 3.747, "read": 3.713}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.572, "count": 111, "max_ms": 12.914, "p95_ms": 10.985}, "gstreamer_push": {"avg_ms": 0.914, "count": 112, "max_ms": 2.937, "p95_ms": 1.001}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.888, "count": 111, "max_ms": 70.418, "p95_ms": 62.438}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.234, "count": 111, "max_ms": 98.738, "p95_ms": 86.972}, "yolo11_640_fire_11class": {"avg_ms": 53.881, "count": 111, "max_ms": 75.546, "p95_ms": 66.155}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.841, "count": 111, "max_ms": 95.978, "p95_ms": 86.692}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.366, "count": 112, "max_ms": 295.208, "p95_ms": 249.442}, "read_interval": {"avg_ms": 230.386, "count": 111, "max_ms": 305.625, "p95_ms": 258.955}, "triton_wait": {"avg_ms": 69.457, "count": 111, "max_ms": 98.751, "p95_ms": 87.457}, "visualize": {"avg_ms": 10.606, "count": 111, "max_ms": 16.691, "p95_ms": 14.379}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 655}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4055.195, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559533f54ec0] error while decoding MB 79 17, bytestream -18 [h264 @ 0x559534c094c0] error while decoding MB 27 25, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 142, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:11:44+09:00", "last_error_summary": "[h264 @ 0x559533f54ec0] error while decoding MB 79 17, bytestream -18 [h264 @ 0x559534c094c0] error while decoding MB 27 25, bytestream -5", "last_exit_at": "2026-07-01T20:09:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.987, "last_started_at": "2026-07-01T20:09:55+09:00", "pid": 21358, "process": {"exists": false, "pid": 21358}, "restart_count": 165, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 166}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.465, "sample_count": 1254, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78605, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:11:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T20:11:44+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T20:11:44+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-01T20:11:44.759683+09:00", "mtime_epoch": 1782904304.7596827, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181351}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:11:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 890.555, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.353, "pushed": 4.353, "read": 4.353}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.529, "count": 129, "max_ms": 10.699, "p95_ms": 9.741}, "gstreamer_push": {"avg_ms": 0.903, "count": 130, "max_ms": 1.889, "p95_ms": 1.24}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.821, "count": 129, "max_ms": 70.588, "p95_ms": 64.141}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.104, "count": 129, "max_ms": 89.904, "p95_ms": 87.663}, "yolo11_640_fire_11class": {"avg_ms": 53.967, "count": 129, "max_ms": 70.046, "p95_ms": 66.113}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.708, "count": 129, "max_ms": 90.649, "p95_ms": 86.719}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.439, "count": 130, "max_ms": 300.271, "p95_ms": 238.882}, "read_interval": {"avg_ms": 231.064, "count": 130, "max_ms": 310.384, "p95_ms": 248.019}, "triton_wait": {"avg_ms": 69.538, "count": 129, "max_ms": 90.661, "p95_ms": 88.501}, "visualize": {"avg_ms": 10.187, "count": 129, "max_ms": 11.903, "p95_ms": 11.34}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.863, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 655}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4060.656, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:11:44+09:00", "last_error_summary": "[h264 @ 0x559533f54ec0] error while decoding MB 79 17, bytestream -18 [h264 @ 0x559534c094c0] error while decoding MB 27 25, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 142, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:11:44+09:00", "last_error_summary": "[h264 @ 0x559533f54ec0] error while decoding MB 79 17, bytestream -18 [h264 @ 0x559534c094c0] error while decoding MB 27 25, bytestream -5", "last_exit_at": "2026-07-01T20:11:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.168, "last_started_at": "2026-07-01T20:09:55+09:00", "pid": 21358, "process": {"exists": false, "pid": 21358}, "restart_count": 165, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 166}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.465, "sample_count": 1254, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78606, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:11:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T20:11:44+09:00", "last_frame_read_age_sec": 0.234, "last_frame_read_at": "2026-07-01T20:11:44+09:00", "mjpeg": {"age_sec": 0.219, "exists": true, "mtime": "2026-07-01T20:11:44.759683+09:00", "mtime_epoch": 1782904304.7596827, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181351}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:11:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 896.48, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.361, "pushed": 4.361, "read": 4.328}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.528, "count": 130, "max_ms": 10.699, "p95_ms": 9.741}, "gstreamer_push": {"avg_ms": 0.903, "count": 130, "max_ms": 1.889, "p95_ms": 1.24}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.768, "count": 130, "max_ms": 70.588, "p95_ms": 64.141}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.098, "count": 130, "max_ms": 89.904, "p95_ms": 87.663}, "yolo11_640_fire_11class": {"avg_ms": 53.946, "count": 130, "max_ms": 70.046, "p95_ms": 66.113}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.672, "count": 130, "max_ms": 90.649, "p95_ms": 86.719}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.441, "count": 130, "max_ms": 300.271, "p95_ms": 238.882}, "read_interval": {"avg_ms": 231.016, "count": 129, "max_ms": 310.384, "p95_ms": 248.019}, "triton_wait": {"avg_ms": 69.529, "count": 130, "max_ms": 90.661, "p95_ms": 88.501}, "visualize": {"avg_ms": 10.187, "count": 130, "max_ms": 11.903, "p95_ms": 11.34}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.809, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 655}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4060.648, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 142, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:11:44+09:00", "last_error_summary": "[h264 @ 0x559533f54ec0] error while decoding MB 79 17, bytestream -18 [h264 @ 0x559534c094c0] error while decoding MB 27 25, bytestream -5", "last_exit_at": "2026-07-01T20:11:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.171, "last_started_at": "2026-07-01T20:09:55+09:00", "pid": 21358, "process": {"exists": false, "pid": 21358}, "restart_count": 165, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 166}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.465, "sample_count": 1254, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78606, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:11:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T20:11:44+09:00", "last_frame_read_age_sec": 0.237, "last_frame_read_at": "2026-07-01T20:11:44+09:00", "mjpeg": {"age_sec": 0.222, "exists": true, "mtime": "2026-07-01T20:11:44.759683+09:00", "mtime_epoch": 1782904304.7596827, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181351}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:11:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 896.48, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.361, "pushed": 4.361, "read": 4.327}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.528, "count": 130, "max_ms": 10.699, "p95_ms": 9.741}, "gstreamer_push": {"avg_ms": 0.903, "count": 130, "max_ms": 1.889, "p95_ms": 1.24}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.768, "count": 130, "max_ms": 70.588, "p95_ms": 64.141}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.098, "count": 130, "max_ms": 89.904, "p95_ms": 87.663}, "yolo11_640_fire_11class": {"avg_ms": 53.946, "count": 130, "max_ms": 70.046, "p95_ms": 66.113}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.672, "count": 130, "max_ms": 90.649, "p95_ms": 86.719}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.441, "count": 130, "max_ms": 300.271, "p95_ms": 238.882}, "read_interval": {"avg_ms": 231.016, "count": 129, "max_ms": 310.384, "p95_ms": 248.019}, "triton_wait": {"avg_ms": 69.529, "count": 130, "max_ms": 90.661, "p95_ms": 88.501}, "visualize": {"avg_ms": 10.187, "count": 130, "max_ms": 11.903, "p95_ms": 11.34}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.812, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 655}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4168.992, "limit_mb": 8796093022207.996}, "detail": {"pid": 21465, "restart_count": 166, "start_count": 167}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 142, "h264_decode_error_count": 148, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:11:44+09:00", "last_error_summary": "[h264 @ 0x559533f54ec0] error while decoding MB 79 17, bytestream -18 [h264 @ 0x559534c094c0] error while decoding MB 27 25, bytestream -5", "last_exit_at": "2026-07-01T20:11:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:11:47+09:00", "pid": 21465, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.156, "name": "ffmpeg", "pid": 21465, "rss_mb": 139.156, "state": "S (sleeping)", "threads": 106}, "restart_count": 166, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 167}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.465, "sample_count": 1233, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78606, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:11:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 5.005, "last_frame_pushed_at": "2026-07-01T20:11:44+09:00", "last_frame_read_age_sec": 5.234, "last_frame_read_at": "2026-07-01T20:11:44+09:00", "mjpeg": {"age_sec": 4.987, "exists": true, "mtime": "2026-07-01T20:11:44.991688+09:00", "mtime_epoch": 1782904304.9916885, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180968}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:11:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 896.48, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.609, "pushed": 3.609, "read": 3.576}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.612, "count": 107, "max_ms": 10.699, "p95_ms": 9.793}, "gstreamer_push": {"avg_ms": 0.911, "count": 108, "max_ms": 1.889, "p95_ms": 1.362}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.576, "count": 107, "max_ms": 70.588, "p95_ms": 63.827}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.025, "count": 107, "max_ms": 89.904, "p95_ms": 87.591}, "yolo11_640_fire_11class": {"avg_ms": 53.662, "count": 107, "max_ms": 70.046, "p95_ms": 66.113}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.216, "count": 107, "max_ms": 90.649, "p95_ms": 86.033}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.152, "count": 108, "max_ms": 300.271, "p95_ms": 239.968}, "read_interval": {"avg_ms": 232.733, "count": 107, "max_ms": 310.384, "p95_ms": 249.288}, "triton_wait": {"avg_ms": 69.281, "count": 107, "max_ms": 90.661, "p95_ms": 88.019}, "visualize": {"avg_ms": 10.265, "count": 107, "max_ms": 11.903, "p95_ms": 11.34}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4064.586, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5599c5712980] error while decoding MB 0 43, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 149, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:13:37+09:00", "last_error_summary": "[h264 @ 0x5599c5712980] error while decoding MB 0 43, bytestream -5", "last_exit_at": "2026-07-01T20:11:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.057, "last_started_at": "2026-07-01T20:11:47+09:00", "pid": 21465, "process": {"exists": false, "pid": 21465}, "restart_count": 166, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 167}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.45, "sample_count": 1240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79063, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:13:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T20:13:36+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T20:13:36+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T20:13:36.994388+09:00", "mtime_epoch": 1782904416.994388, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179243}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:13:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 902.051, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.292, "pushed": 4.292, "read": 4.292}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.755, "count": 128, "max_ms": 11.58, "p95_ms": 10.269}, "gstreamer_push": {"avg_ms": 0.902, "count": 128, "max_ms": 1.843, "p95_ms": 1.018}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.859, "count": 128, "max_ms": 71.525, "p95_ms": 62.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.457, "count": 128, "max_ms": 91.658, "p95_ms": 85.094}, "yolo11_640_fire_11class": {"avg_ms": 53.719, "count": 128, "max_ms": 70.757, "p95_ms": 68.44}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.152, "count": 128, "max_ms": 91.895, "p95_ms": 88.498}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.697, "count": 128, "max_ms": 321.989, "p95_ms": 253.833}, "read_interval": {"avg_ms": 234.323, "count": 128, "max_ms": 333.506, "p95_ms": 264.046}, "triton_wait": {"avg_ms": 68.981, "count": 128, "max_ms": 91.906, "p95_ms": 88.51}, "visualize": {"avg_ms": 10.397, "count": 128, "max_ms": 15.384, "p95_ms": 11.485}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.82, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4063.664, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:13:37+09:00", "last_error_summary": "[h264 @ 0x5599c5712980] error while decoding MB 0 43, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 149, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:13:37+09:00", "last_error_summary": "[h264 @ 0x5599c5712980] error while decoding MB 0 43, bytestream -5", "last_exit_at": "2026-07-01T20:13:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.212, "last_started_at": "2026-07-01T20:11:47+09:00", "pid": 21465, "process": {"exists": false, "pid": 21465}, "restart_count": 166, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 167}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.45, "sample_count": 1240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79064, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:13:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T20:13:37+09:00", "last_frame_read_age_sec": 0.206, "last_frame_read_at": "2026-07-01T20:13:36+09:00", "mjpeg": {"age_sec": 0.196, "exists": true, "mtime": "2026-07-01T20:13:36.994388+09:00", "mtime_epoch": 1782904416.994388, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179243}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:13:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 902.051, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.304, "pushed": 4.304, "read": 4.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.756, "count": 128, "max_ms": 11.58, "p95_ms": 10.269}, "gstreamer_push": {"avg_ms": 0.902, "count": 129, "max_ms": 1.843, "p95_ms": 1.018}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.691, "count": 128, "max_ms": 71.525, "p95_ms": 62.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.303, "count": 128, "max_ms": 91.658, "p95_ms": 85.094}, "yolo11_640_fire_11class": {"avg_ms": 53.595, "count": 128, "max_ms": 70.757, "p95_ms": 68.44}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.006, "count": 128, "max_ms": 91.895, "p95_ms": 88.498}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.517, "count": 129, "max_ms": 321.989, "p95_ms": 253.833}, "read_interval": {"avg_ms": 234.323, "count": 128, "max_ms": 333.506, "p95_ms": 264.046}, "triton_wait": {"avg_ms": 68.827, "count": 128, "max_ms": 91.906, "p95_ms": 88.51}, "visualize": {"avg_ms": 10.393, "count": 128, "max_ms": 15.384, "p95_ms": 11.485}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4063.781, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 149, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:13:37+09:00", "last_error_summary": "[h264 @ 0x5599c5712980] error while decoding MB 0 43, bytestream -5", "last_exit_at": "2026-07-01T20:13:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.216, "last_started_at": "2026-07-01T20:11:47+09:00", "pid": 21465, "process": {"exists": false, "pid": 21465}, "restart_count": 166, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 167}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.45, "sample_count": 1240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79064, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:13:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:13:37+09:00", "last_frame_read_age_sec": 0.209, "last_frame_read_at": "2026-07-01T20:13:36+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-01T20:13:36.994388+09:00", "mtime_epoch": 1782904416.994388, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179243}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:13:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 902.051, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.303, "pushed": 4.303, "read": 4.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.756, "count": 128, "max_ms": 11.58, "p95_ms": 10.269}, "gstreamer_push": {"avg_ms": 0.902, "count": 129, "max_ms": 1.843, "p95_ms": 1.018}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.691, "count": 128, "max_ms": 71.525, "p95_ms": 62.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.303, "count": 128, "max_ms": 91.658, "p95_ms": 85.094}, "yolo11_640_fire_11class": {"avg_ms": 53.595, "count": 128, "max_ms": 70.757, "p95_ms": 68.44}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.006, "count": 128, "max_ms": 91.895, "p95_ms": 88.498}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.517, "count": 129, "max_ms": 321.989, "p95_ms": 253.833}, "read_interval": {"avg_ms": 234.323, "count": 128, "max_ms": 333.506, "p95_ms": 264.046}, "triton_wait": {"avg_ms": 68.827, "count": 128, "max_ms": 91.906, "p95_ms": 88.51}, "visualize": {"avg_ms": 10.393, "count": 128, "max_ms": 15.384, "p95_ms": 11.485}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4174.461, "limit_mb": 8796093022207.996}, "detail": {"pid": 21572, "restart_count": 167, "start_count": 168}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 143, "h264_decode_error_count": 149, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:13:37+09:00", "last_error_summary": "[h264 @ 0x5599c5712980] error while decoding MB 0 43, bytestream -5", "last_exit_at": "2026-07-01T20:13:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:13:39+09:00", "pid": 21572, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.152, "name": "ffmpeg", "pid": 21572, "rss_mb": 139.152, "state": "S (sleeping)", "threads": 106}, "restart_count": 167, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 168}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.45, "sample_count": 1221, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79064, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:13:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.678, "last_frame_pushed_at": "2026-07-01T20:13:37+09:00", "last_frame_read_age_sec": 4.881, "last_frame_read_at": "2026-07-01T20:13:36+09:00", "mjpeg": {"age_sec": 4.663, "exists": true, "mtime": "2026-07-01T20:13:37.202393+09:00", "mtime_epoch": 1782904417.202393, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179035}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:13:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 902.051, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.626, "pushed": 3.626, "read": 3.592}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.761, "count": 108, "max_ms": 11.58, "p95_ms": 10.269}, "gstreamer_push": {"avg_ms": 0.908, "count": 108, "max_ms": 1.843, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.907, "count": 108, "max_ms": 71.525, "p95_ms": 62.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.876, "count": 108, "max_ms": 91.108, "p95_ms": 85.094}, "yolo11_640_fire_11class": {"avg_ms": 53.399, "count": 108, "max_ms": 70.567, "p95_ms": 68.058}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.786, "count": 108, "max_ms": 91.895, "p95_ms": 84.545}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.147, "count": 108, "max_ms": 321.989, "p95_ms": 253.833}, "read_interval": {"avg_ms": 234.978, "count": 107, "max_ms": 333.506, "p95_ms": 264.046}, "triton_wait": {"avg_ms": 68.385, "count": 108, "max_ms": 91.906, "p95_ms": 86.88}, "visualize": {"avg_ms": 10.457, "count": 108, "max_ms": 15.384, "p95_ms": 11.668}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.786, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 662}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4068.434, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f6fcfd6880] error while decoding MB 13 42, bytestream -42", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 144, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:15:27+09:00", "last_error_summary": "[h264 @ 0x55f6fcfd6880] error while decoding MB 13 42, bytestream -42", "last_exit_at": "2026-07-01T20:13:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.349, "last_started_at": "2026-07-01T20:13:39+09:00", "pid": 21572, "process": {"exists": false, "pid": 21572}, "restart_count": 167, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 168}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.816, "sample_count": 1242, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79519, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:15:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T20:15:27+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T20:15:27+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T20:15:27.168928+09:00", "mtime_epoch": 1782904527.1689277, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176270}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:15:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 902.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.381, "pushed": 4.381, "read": 4.381}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.495, "count": 130, "max_ms": 11.435, "p95_ms": 10.052}, "gstreamer_push": {"avg_ms": 0.91, "count": 131, "max_ms": 1.878, "p95_ms": 1.067}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.455, "count": 130, "max_ms": 70.806, "p95_ms": 63.579}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.45, "count": 130, "max_ms": 94.031, "p95_ms": 89.863}, "yolo11_640_fire_11class": {"avg_ms": 54.294, "count": 130, "max_ms": 76.052, "p95_ms": 69.854}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.911, "count": 130, "max_ms": 96.376, "p95_ms": 88.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.907, "count": 131, "max_ms": 284.057, "p95_ms": 254.585}, "read_interval": {"avg_ms": 229.551, "count": 131, "max_ms": 294.482, "p95_ms": 263.825}, "triton_wait": {"avg_ms": 69.249, "count": 130, "max_ms": 96.39, "p95_ms": 90.611}, "visualize": {"avg_ms": 10.11, "count": 130, "max_ms": 12.465, "p95_ms": 11.334}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 662}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4067.336, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:15:27+09:00", "last_error_summary": "[h264 @ 0x55f6fcfd6880] error while decoding MB 13 42, bytestream -42", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 144, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:15:27+09:00", "last_error_summary": "[h264 @ 0x55f6fcfd6880] error while decoding MB 13 42, bytestream -42", "last_exit_at": "2026-07-01T20:15:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.516, "last_started_at": "2026-07-01T20:13:39+09:00", "pid": 21572, "process": {"exists": false, "pid": 21572}, "restart_count": 167, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 168}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.816, "sample_count": 1243, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79520, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:15:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T20:15:27+09:00", "last_frame_read_age_sec": 0.221, "last_frame_read_at": "2026-07-01T20:15:27+09:00", "mjpeg": {"age_sec": 0.211, "exists": true, "mtime": "2026-07-01T20:15:27.168928+09:00", "mtime_epoch": 1782904527.1689277, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176270}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:15:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 902.5, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.39, "pushed": 4.39, "read": 4.356}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.494, "count": 131, "max_ms": 11.435, "p95_ms": 10.052}, "gstreamer_push": {"avg_ms": 0.91, "count": 131, "max_ms": 1.878, "p95_ms": 1.067}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.46, "count": 131, "max_ms": 70.806, "p95_ms": 63.579}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.481, "count": 131, "max_ms": 94.031, "p95_ms": 89.863}, "yolo11_640_fire_11class": {"avg_ms": 54.322, "count": 131, "max_ms": 76.052, "p95_ms": 69.854}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.905, "count": 131, "max_ms": 96.376, "p95_ms": 88.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.859, "count": 131, "max_ms": 284.057, "p95_ms": 254.585}, "read_interval": {"avg_ms": 229.536, "count": 130, "max_ms": 294.482, "p95_ms": 263.825}, "triton_wait": {"avg_ms": 69.275, "count": 131, "max_ms": 96.39, "p95_ms": 90.611}, "visualize": {"avg_ms": 10.114, "count": 131, "max_ms": 12.465, "p95_ms": 11.334}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.841, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 662}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4067.281, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 144, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:15:27+09:00", "last_error_summary": "[h264 @ 0x55f6fcfd6880] error while decoding MB 13 42, bytestream -42", "last_exit_at": "2026-07-01T20:15:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.519, "last_started_at": "2026-07-01T20:13:39+09:00", "pid": 21572, "process": {"exists": false, "pid": 21572}, "restart_count": 167, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 168}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.816, "sample_count": 1243, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79520, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:15:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T20:15:27+09:00", "last_frame_read_age_sec": 0.224, "last_frame_read_at": "2026-07-01T20:15:27+09:00", "mjpeg": {"age_sec": 0.215, "exists": true, "mtime": "2026-07-01T20:15:27.168928+09:00", "mtime_epoch": 1782904527.1689277, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176270}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:15:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 902.5, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.39, "pushed": 4.39, "read": 4.356}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.494, "count": 131, "max_ms": 11.435, "p95_ms": 10.052}, "gstreamer_push": {"avg_ms": 0.91, "count": 131, "max_ms": 1.878, "p95_ms": 1.067}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.46, "count": 131, "max_ms": 70.806, "p95_ms": 63.579}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.481, "count": 131, "max_ms": 94.031, "p95_ms": 89.863}, "yolo11_640_fire_11class": {"avg_ms": 54.322, "count": 131, "max_ms": 76.052, "p95_ms": 69.854}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.905, "count": 131, "max_ms": 96.376, "p95_ms": 88.99}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.859, "count": 131, "max_ms": 284.057, "p95_ms": 254.585}, "read_interval": {"avg_ms": 229.536, "count": 130, "max_ms": 294.482, "p95_ms": 263.825}, "triton_wait": {"avg_ms": 69.275, "count": 131, "max_ms": 96.39, "p95_ms": 90.611}, "visualize": {"avg_ms": 10.114, "count": 131, "max_ms": 12.465, "p95_ms": 11.334}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.844, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 662}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4178.895, "limit_mb": 8796093022207.996}, "detail": {"pid": 21679, "restart_count": 168, "start_count": 169}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 144, "h264_decode_error_count": 150, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:15:27+09:00", "last_error_summary": "[h264 @ 0x55f6fcfd6880] error while decoding MB 13 42, bytestream -42", "last_exit_at": "2026-07-01T20:15:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:15:29+09:00", "pid": 21679, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.055, "name": "ffmpeg", "pid": 21679, "rss_mb": 142.055, "state": "S (sleeping)", "threads": 106}, "restart_count": 168, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 169}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.735, "sample_count": 1225, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79520, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:15:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.444, "last_frame_pushed_at": "2026-07-01T20:15:27+09:00", "last_frame_read_age_sec": 4.662, "last_frame_read_at": "2026-07-01T20:15:27+09:00", "mjpeg": {"age_sec": 4.428, "exists": true, "mtime": "2026-07-01T20:15:27.392933+09:00", "mtime_epoch": 1782904527.392933, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173189}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:15:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 902.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.746, "pushed": 3.746, "read": 3.712}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.41, "count": 111, "max_ms": 11.435, "p95_ms": 10.052}, "gstreamer_push": {"avg_ms": 0.909, "count": 112, "max_ms": 1.878, "p95_ms": 1.067}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.653, "count": 111, "max_ms": 70.806, "p95_ms": 63.727}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.066, "count": 111, "max_ms": 94.031, "p95_ms": 90.599}, "yolo11_640_fire_11class": {"avg_ms": 54.674, "count": 111, "max_ms": 76.052, "p95_ms": 70.472}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.431, "count": 111, "max_ms": 96.376, "p95_ms": 89.092}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.581, "count": 112, "max_ms": 284.057, "p95_ms": 254.585}, "read_interval": {"avg_ms": 229.279, "count": 111, "max_ms": 294.482, "p95_ms": 263.825}, "triton_wait": {"avg_ms": 69.931, "count": 111, "max_ms": 96.39, "p95_ms": 90.734}, "visualize": {"avg_ms": 10.079, "count": 111, "max_ms": 12.465, "p95_ms": 11.278}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 666}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4066.207, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559e3b47d480] error while decoding MB 21 39, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 151, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:17:17+09:00", "last_error_summary": "[h264 @ 0x559e3b47d480] error while decoding MB 21 39, bytestream -5", "last_exit_at": "2026-07-01T20:15:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.176, "last_started_at": "2026-07-01T20:15:29+09:00", "pid": 21679, "process": {"exists": false, "pid": 21679}, "restart_count": 168, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 169}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.88, "sample_count": 1255, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79981, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:17:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-01T20:17:17+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T20:17:17+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-01T20:17:17.955388+09:00", "mtime_epoch": 1782904637.9553878, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165202}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:17:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.261, "pushed": 4.261, "read": 4.261}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.827, "count": 127, "max_ms": 13.175, "p95_ms": 10.898}, "gstreamer_push": {"avg_ms": 0.95, "count": 127, "max_ms": 2.947, "p95_ms": 1.842}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.531, "count": 127, "max_ms": 72.287, "p95_ms": 61.998}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.194, "count": 127, "max_ms": 94.542, "p95_ms": 89.247}, "yolo11_640_fire_11class": {"avg_ms": 54.329, "count": 127, "max_ms": 80.36, "p95_ms": 67.546}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.448, "count": 127, "max_ms": 95.658, "p95_ms": 87.439}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.273, "count": 127, "max_ms": 323.19, "p95_ms": 254.939}, "read_interval": {"avg_ms": 235.978, "count": 127, "max_ms": 336.29, "p95_ms": 264.863}, "triton_wait": {"avg_ms": 69.639, "count": 127, "max_ms": 95.671, "p95_ms": 89.358}, "visualize": {"avg_ms": 10.617, "count": 127, "max_ms": 15.147, "p95_ms": 13.228}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.809, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 666}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4065.398, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:17:17+09:00", "last_error_summary": "[h264 @ 0x559e3b47d480] error while decoding MB 21 39, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 151, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:17:17+09:00", "last_error_summary": "[h264 @ 0x559e3b47d480] error while decoding MB 21 39, bytestream -5", "last_exit_at": "2026-07-01T20:17:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.336, "last_started_at": "2026-07-01T20:15:29+09:00", "pid": 21679, "process": {"exists": false, "pid": 21679}, "restart_count": 168, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 169}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.82, "sample_count": 1255, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79982, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:17:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T20:17:18+09:00", "last_frame_read_age_sec": 0.211, "last_frame_read_at": "2026-07-01T20:17:17+09:00", "mjpeg": {"age_sec": 0.201, "exists": true, "mtime": "2026-07-01T20:17:17.955388+09:00", "mtime_epoch": 1782904637.9553878, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165202}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:17:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.672, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.271, "pushed": 4.271, "read": 4.238}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.827, "count": 127, "max_ms": 13.175, "p95_ms": 10.898}, "gstreamer_push": {"avg_ms": 0.95, "count": 128, "max_ms": 2.947, "p95_ms": 1.842}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.554, "count": 127, "max_ms": 72.287, "p95_ms": 61.998}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.183, "count": 127, "max_ms": 94.542, "p95_ms": 89.247}, "yolo11_640_fire_11class": {"avg_ms": 54.351, "count": 127, "max_ms": 80.36, "p95_ms": 67.546}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.441, "count": 127, "max_ms": 95.658, "p95_ms": 87.439}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.121, "count": 128, "max_ms": 323.19, "p95_ms": 254.939}, "read_interval": {"avg_ms": 235.978, "count": 127, "max_ms": 336.29, "p95_ms": 264.863}, "triton_wait": {"avg_ms": 69.627, "count": 127, "max_ms": 95.671, "p95_ms": 89.358}, "visualize": {"avg_ms": 10.615, "count": 127, "max_ms": 15.147, "p95_ms": 13.228}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 666}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4065.344, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 151, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:17:17+09:00", "last_error_summary": "[h264 @ 0x559e3b47d480] error while decoding MB 21 39, bytestream -5", "last_exit_at": "2026-07-01T20:17:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.34, "last_started_at": "2026-07-01T20:15:29+09:00", "pid": 21679, "process": {"exists": false, "pid": 21679}, "restart_count": 168, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 169}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.82, "sample_count": 1255, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79982, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:17:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:17:18+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-01T20:17:17+09:00", "mjpeg": {"age_sec": 0.204, "exists": true, "mtime": "2026-07-01T20:17:17.955388+09:00", "mtime_epoch": 1782904637.9553878, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165202}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:17:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.672, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.271, "pushed": 4.271, "read": 4.237}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.827, "count": 127, "max_ms": 13.175, "p95_ms": 10.898}, "gstreamer_push": {"avg_ms": 0.95, "count": 128, "max_ms": 2.947, "p95_ms": 1.842}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.554, "count": 127, "max_ms": 72.287, "p95_ms": 61.998}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.183, "count": 127, "max_ms": 94.542, "p95_ms": 89.247}, "yolo11_640_fire_11class": {"avg_ms": 54.351, "count": 127, "max_ms": 80.36, "p95_ms": 67.546}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.441, "count": 127, "max_ms": 95.658, "p95_ms": 87.439}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.121, "count": 128, "max_ms": 323.19, "p95_ms": 254.939}, "read_interval": {"avg_ms": 235.978, "count": 127, "max_ms": 336.29, "p95_ms": 264.863}, "triton_wait": {"avg_ms": 69.627, "count": 127, "max_ms": 95.671, "p95_ms": 89.358}, "visualize": {"avg_ms": 10.615, "count": 127, "max_ms": 15.147, "p95_ms": 13.228}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 666}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4173.051, "limit_mb": 8796093022207.996}, "detail": {"pid": 21786, "restart_count": 169, "start_count": 170}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 145, "h264_decode_error_count": 151, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:17:17+09:00", "last_error_summary": "[h264 @ 0x559e3b47d480] error while decoding MB 21 39, bytestream -5", "last_exit_at": "2026-07-01T20:17:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:17:20+09:00", "pid": 21786, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.164, "name": "ffmpeg", "pid": 21786, "rss_mb": 139.164, "state": "S (sleeping)", "threads": 106}, "restart_count": 169, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 170}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.88, "sample_count": 1236, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79982, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:17:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.3, "last_frame_pushed_at": "2026-07-01T20:17:18+09:00", "last_frame_read_age_sec": 4.508, "last_frame_read_at": "2026-07-01T20:17:17+09:00", "mjpeg": {"age_sec": 4.282, "exists": true, "mtime": "2026-07-01T20:17:18.171392+09:00", "mtime_epoch": 1782904638.1713924, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165941}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:17:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 899.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.684, "pushed": 3.684, "read": 3.65}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.719, "count": 109, "max_ms": 11.531, "p95_ms": 10.894}, "gstreamer_push": {"avg_ms": 0.941, "count": 110, "max_ms": 2.947, "p95_ms": 1.842}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.519, "count": 109, "max_ms": 72.287, "p95_ms": 62.292}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.617, "count": 109, "max_ms": 94.542, "p95_ms": 89.738}, "yolo11_640_fire_11class": {"avg_ms": 54.833, "count": 109, "max_ms": 80.36, "p95_ms": 68.055}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.986, "count": 109, "max_ms": 95.658, "p95_ms": 89.346}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.915, "count": 110, "max_ms": 323.19, "p95_ms": 254.939}, "read_interval": {"avg_ms": 234.731, "count": 109, "max_ms": 336.29, "p95_ms": 264.863}, "triton_wait": {"avg_ms": 70.05, "count": 109, "max_ms": 95.671, "p95_ms": 89.751}, "visualize": {"avg_ms": 10.382, "count": 109, "max_ms": 14.164, "p95_ms": 12.047}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.859, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 671}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4061.871, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559efeb048c0] error while decoding MB 76 41, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 152, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:19:40+09:00", "last_error_summary": "[h264 @ 0x559efeb048c0] error while decoding MB 76 41, bytestream -13", "last_exit_at": "2026-07-01T20:17:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.073, "last_started_at": "2026-07-01T20:17:20+09:00", "pid": 21786, "process": {"exists": false, "pid": 21786}, "restart_count": 169, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 170}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.2, "sample_count": 1243, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80559, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:19:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-01T20:19:40+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T20:19:40+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-01T20:19:40.482444+09:00", "mtime_epoch": 1782904780.4824438, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161410}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:19:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 895.113, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.1, "pushed": 4.1, "read": 4.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.277, "count": 123, "max_ms": 13.405, "p95_ms": 10.428}, "gstreamer_push": {"avg_ms": 0.933, "count": 123, "max_ms": 2.921, "p95_ms": 1.003}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.384, "count": 123, "max_ms": 69.126, "p95_ms": 62.217}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.378, "count": 123, "max_ms": 90.664, "p95_ms": 86.025}, "yolo11_640_fire_11class": {"avg_ms": 53.023, "count": 123, "max_ms": 71.194, "p95_ms": 66.559}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.439, "count": 123, "max_ms": 92.39, "p95_ms": 87.105}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.76, "count": 123, "max_ms": 313.3, "p95_ms": 259.966}, "read_interval": {"avg_ms": 243.334, "count": 124, "max_ms": 323.953, "p95_ms": 270.534}, "triton_wait": {"avg_ms": 67.931, "count": 123, "max_ms": 92.403, "p95_ms": 88.383}, "visualize": {"avg_ms": 11.024, "count": 123, "max_ms": 15.002, "p95_ms": 12.35}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 671}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4061.105, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:19:40+09:00", "last_error_summary": "[h264 @ 0x559efeb048c0] error while decoding MB 76 41, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 152, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:19:40+09:00", "last_error_summary": "[h264 @ 0x559efeb048c0] error while decoding MB 76 41, bytestream -13", "last_exit_at": "2026-07-01T20:19:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.283, "last_started_at": "2026-07-01T20:17:20+09:00", "pid": 21786, "process": {"exists": false, "pid": 21786}, "restart_count": 169, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 170}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.235, "sample_count": 1243, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80560, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:19:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T20:19:40+09:00", "last_frame_read_age_sec": 0.263, "last_frame_read_at": "2026-07-01T20:19:40+09:00", "mjpeg": {"age_sec": 0.252, "exists": true, "mtime": "2026-07-01T20:19:40.482444+09:00", "mtime_epoch": 1782904780.4824438, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161410}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:19:40+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 895.113, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.136, "pushed": 4.136, "read": 4.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.278, "count": 123, "max_ms": 13.405, "p95_ms": 10.428}, "gstreamer_push": {"avg_ms": 0.933, "count": 124, "max_ms": 2.921, "p95_ms": 1.003}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.406, "count": 123, "max_ms": 69.126, "p95_ms": 62.217}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.559, "count": 123, "max_ms": 90.664, "p95_ms": 86.406}, "yolo11_640_fire_11class": {"avg_ms": 53.128, "count": 123, "max_ms": 71.194, "p95_ms": 67.334}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.645, "count": 123, "max_ms": 92.39, "p95_ms": 87.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.95, "count": 124, "max_ms": 313.3, "p95_ms": 259.966}, "read_interval": {"avg_ms": 243.437, "count": 123, "max_ms": 323.953, "p95_ms": 270.534}, "triton_wait": {"avg_ms": 68.137, "count": 123, "max_ms": 92.403, "p95_ms": 88.658}, "visualize": {"avg_ms": 11.027, "count": 123, "max_ms": 15.002, "p95_ms": 12.35}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 671}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4061.098, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 152, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:19:40+09:00", "last_error_summary": "[h264 @ 0x559efeb048c0] error while decoding MB 76 41, bytestream -13", "last_exit_at": "2026-07-01T20:19:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 140.286, "last_started_at": "2026-07-01T20:17:20+09:00", "pid": 21786, "process": {"exists": false, "pid": 21786}, "restart_count": 169, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 170}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.235, "sample_count": 1243, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80560, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:19:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T20:19:40+09:00", "last_frame_read_age_sec": 0.266, "last_frame_read_at": "2026-07-01T20:19:40+09:00", "mjpeg": {"age_sec": 0.255, "exists": true, "mtime": "2026-07-01T20:19:40.482444+09:00", "mtime_epoch": 1782904780.4824438, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161410}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:19:40+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 895.113, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.136, "pushed": 4.136, "read": 4.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.278, "count": 123, "max_ms": 13.405, "p95_ms": 10.428}, "gstreamer_push": {"avg_ms": 0.933, "count": 124, "max_ms": 2.921, "p95_ms": 1.003}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.406, "count": 123, "max_ms": 69.126, "p95_ms": 62.217}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.559, "count": 123, "max_ms": 90.664, "p95_ms": 86.406}, "yolo11_640_fire_11class": {"avg_ms": 53.128, "count": 123, "max_ms": 71.194, "p95_ms": 67.334}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.645, "count": 123, "max_ms": 92.39, "p95_ms": 87.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.95, "count": 124, "max_ms": 313.3, "p95_ms": 259.966}, "read_interval": {"avg_ms": 243.437, "count": 123, "max_ms": 323.953, "p95_ms": 270.534}, "triton_wait": {"avg_ms": 68.137, "count": 123, "max_ms": 92.403, "p95_ms": 88.658}, "visualize": {"avg_ms": 11.027, "count": 123, "max_ms": 15.002, "p95_ms": 12.35}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 671}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4171.766, "limit_mb": 8796093022207.996}, "detail": {"pid": 21893, "restart_count": 170, "start_count": 171}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 146, "h264_decode_error_count": 152, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:19:40+09:00", "last_error_summary": "[h264 @ 0x559efeb048c0] error while decoding MB 76 41, bytestream -13", "last_exit_at": "2026-07-01T20:19:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:19:43+09:00", "pid": 21893, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.918, "name": "ffmpeg", "pid": 21893, "rss_mb": 138.918, "state": "S (sleeping)", "threads": 106}, "restart_count": 170, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 171}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.29, "sample_count": 1221, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80560, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:19:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.769, "last_frame_pushed_at": "2026-07-01T20:19:40+09:00", "last_frame_read_age_sec": 5.028, "last_frame_read_at": "2026-07-01T20:19:40+09:00", "mjpeg": {"age_sec": 4.753, "exists": true, "mtime": "2026-07-01T20:19:40.746449+09:00", "mtime_epoch": 1782904780.7464492, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 161703}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:19:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 895.113, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.489, "pushed": 3.489, "read": 3.456}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.323, "count": 104, "max_ms": 13.405, "p95_ms": 10.473}, "gstreamer_push": {"avg_ms": 0.925, "count": 104, "max_ms": 2.914, "p95_ms": 0.996}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.193, "count": 104, "max_ms": 68.781, "p95_ms": 61.237}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.267, "count": 104, "max_ms": 90.664, "p95_ms": 86.025}, "yolo11_640_fire_11class": {"avg_ms": 52.884, "count": 104, "max_ms": 71.194, "p95_ms": 67.334}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.501, "count": 104, "max_ms": 92.39, "p95_ms": 87.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.391, "count": 104, "max_ms": 292.703, "p95_ms": 259.966}, "read_interval": {"avg_ms": 242.812, "count": 103, "max_ms": 301.822, "p95_ms": 270.534}, "triton_wait": {"avg_ms": 67.879, "count": 104, "max_ms": 92.403, "p95_ms": 88.658}, "visualize": {"avg_ms": 11.011, "count": 104, "max_ms": 15.002, "p95_ms": 12.35}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.806, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4065.746, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fd6b91a580] error while decoding MB 25 33, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 153, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:21:58+09:00", "last_error_summary": "[h264 @ 0x55fd6b91a580] error while decoding MB 25 33, bytestream -5", "last_exit_at": "2026-07-01T20:19:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.248, "last_started_at": "2026-07-01T20:19:43+09:00", "pid": 21893, "process": {"exists": false, "pid": 21893}, "restart_count": 170, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 171}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.716, "sample_count": 1250, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81148, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-01T20:21:58+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-01T20:21:58+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-01T20:21:58.705317+09:00", "mtime_epoch": 1782904918.7053165, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156594}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:21:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.703, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 144, "pushed_frames": 144, "read_frames": 144, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.808, "pushed": 4.808, "read": 4.808}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.684, "count": 143, "max_ms": 11.492, "p95_ms": 9.746}, "gstreamer_push": {"avg_ms": 1.023, "count": 144, "max_ms": 6.374, "p95_ms": 1.425}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.095, "count": 143, "max_ms": 69.746, "p95_ms": 61.009}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.325, "count": 143, "max_ms": 92.276, "p95_ms": 91.135}, "yolo11_640_fire_11class": {"avg_ms": 52.806, "count": 143, "max_ms": 77.888, "p95_ms": 67.834}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.753, "count": 143, "max_ms": 95.507, "p95_ms": 88.511}}, "truncated_count": 0}, "processing_total": {"avg_ms": 199.047, "count": 144, "max_ms": 282.92, "p95_ms": 239.686}, "read_interval": {"avg_ms": 209.225, "count": 144, "max_ms": 294.41, "p95_ms": 249.397}, "triton_wait": {"avg_ms": 67.833, "count": 143, "max_ms": 95.519, "p95_ms": 91.151}, "visualize": {"avg_ms": 9.35, "count": 143, "max_ms": 14.018, "p95_ms": 11.634}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4064.926, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:21:58+09:00", "last_error_summary": "[h264 @ 0x55fd6b91a580] error while decoding MB 25 33, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 153, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:21:58+09:00", "last_error_summary": "[h264 @ 0x55fd6b91a580] error while decoding MB 25 33, bytestream -5", "last_exit_at": "2026-07-01T20:21:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.368, "last_started_at": "2026-07-01T20:19:43+09:00", "pid": 21893, "process": {"exists": false, "pid": 21893}, "restart_count": 170, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 171}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.716, "sample_count": 1251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81149, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T20:21:58+09:00", "last_frame_read_age_sec": 0.169, "last_frame_read_at": "2026-07-01T20:21:58+09:00", "mjpeg": {"age_sec": 0.16, "exists": true, "mtime": "2026-07-01T20:21:58.705317+09:00", "mtime_epoch": 1782904918.7053165, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156594}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:21:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.703, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 144, "pushed_frames": 144, "read_frames": 143, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.823, "pushed": 4.823, "read": 4.79}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.67, "count": 144, "max_ms": 11.492, "p95_ms": 9.746}, "gstreamer_push": {"avg_ms": 1.023, "count": 144, "max_ms": 6.374, "p95_ms": 1.425}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.099, "count": 144, "max_ms": 69.746, "p95_ms": 61.009}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.266, "count": 144, "max_ms": 92.276, "p95_ms": 91.135}, "yolo11_640_fire_11class": {"avg_ms": 52.771, "count": 144, "max_ms": 77.888, "p95_ms": 67.834}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.685, "count": 144, "max_ms": 95.507, "p95_ms": 88.511}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.608, "count": 144, "max_ms": 282.92, "p95_ms": 239.686}, "read_interval": {"avg_ms": 209.032, "count": 143, "max_ms": 294.41, "p95_ms": 249.397}, "triton_wait": {"avg_ms": 67.771, "count": 144, "max_ms": 95.519, "p95_ms": 91.151}, "visualize": {"avg_ms": 9.338, "count": 144, "max_ms": 14.018, "p95_ms": 11.634}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.856, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4064.918, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 153, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:21:58+09:00", "last_error_summary": "[h264 @ 0x55fd6b91a580] error while decoding MB 25 33, bytestream -5", "last_exit_at": "2026-07-01T20:21:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 135.371, "last_started_at": "2026-07-01T20:19:43+09:00", "pid": 21893, "process": {"exists": false, "pid": 21893}, "restart_count": 170, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 171}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.716, "sample_count": 1251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81149, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:21:58+09:00", "last_frame_read_age_sec": 0.172, "last_frame_read_at": "2026-07-01T20:21:58+09:00", "mjpeg": {"age_sec": 0.164, "exists": true, "mtime": "2026-07-01T20:21:58.705317+09:00", "mtime_epoch": 1782904918.7053165, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156594}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:21:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.703, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 144, "pushed_frames": 144, "read_frames": 143, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.823, "pushed": 4.823, "read": 4.789}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.67, "count": 144, "max_ms": 11.492, "p95_ms": 9.746}, "gstreamer_push": {"avg_ms": 1.023, "count": 144, "max_ms": 6.374, "p95_ms": 1.425}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.099, "count": 144, "max_ms": 69.746, "p95_ms": 61.009}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.266, "count": 144, "max_ms": 92.276, "p95_ms": 91.135}, "yolo11_640_fire_11class": {"avg_ms": 52.771, "count": 144, "max_ms": 77.888, "p95_ms": 67.834}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.685, "count": 144, "max_ms": 95.507, "p95_ms": 88.511}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.608, "count": 144, "max_ms": 282.92, "p95_ms": 239.686}, "read_interval": {"avg_ms": 209.032, "count": 143, "max_ms": 294.41, "p95_ms": 249.397}, "triton_wait": {"avg_ms": 67.771, "count": 144, "max_ms": 95.519, "p95_ms": 91.151}, "visualize": {"avg_ms": 9.338, "count": 144, "max_ms": 14.018, "p95_ms": 11.634}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.859, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4181.148, "limit_mb": 8796093022207.996}, "detail": {"pid": 22000, "restart_count": 171, "start_count": 172}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 147, "h264_decode_error_count": 153, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:21:58+09:00", "last_error_summary": "[h264 @ 0x55fd6b91a580] error while decoding MB 25 33, bytestream -5", "last_exit_at": "2026-07-01T20:22:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:22:01+09:00", "pid": 22000, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.902, "name": "ffmpeg", "pid": 22000, "rss_mb": 144.902, "state": "S (sleeping)", "threads": 106}, "restart_count": 171, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 172}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.29, "sample_count": 1229, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81149, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.947, "last_frame_pushed_at": "2026-07-01T20:21:58+09:00", "last_frame_read_age_sec": 5.113, "last_frame_read_at": "2026-07-01T20:21:58+09:00", "mjpeg": {"age_sec": 4.932, "exists": true, "mtime": "2026-07-01T20:21:58.877320+09:00", "mtime_epoch": 1782904918.87732, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156784}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:22:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 897.703, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.088, "pushed": 4.088, "read": 4.055}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.608, "count": 122, "max_ms": 11.492, "p95_ms": 9.746}, "gstreamer_push": {"avg_ms": 1.05, "count": 122, "max_ms": 6.374, "p95_ms": 1.816}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.941, "count": 122, "max_ms": 65.02, "p95_ms": 60.504}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.907, "count": 122, "max_ms": 92.276, "p95_ms": 91.135}, "yolo11_640_fire_11class": {"avg_ms": 52.642, "count": 122, "max_ms": 77.888, "p95_ms": 67.739}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.258, "count": 122, "max_ms": 95.507, "p95_ms": 88.479}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.75, "count": 122, "max_ms": 282.92, "p95_ms": 244.605}, "read_interval": {"avg_ms": 206.28, "count": 121, "max_ms": 294.41, "p95_ms": 253.958}, "triton_wait": {"avg_ms": 67.465, "count": 122, "max_ms": 95.519, "p95_ms": 91.151}, "visualize": {"avg_ms": 9.22, "count": 122, "max_ms": 14.018, "p95_ms": 11.716}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.843, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 682}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4072.465, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a8f8e88c40] error while decoding MB 4 30, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 154, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:24:11+09:00", "last_error_summary": "[h264 @ 0x55a8f8e88c40] error while decoding MB 4 30, bytestream -7", "last_exit_at": "2026-07-01T20:22:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.679, "last_started_at": "2026-07-01T20:22:01+09:00", "pid": 22000, "process": {"exists": false, "pid": 22000}, "restart_count": 171, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 172}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.936, "sample_count": 1405, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81845, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:24:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-01T20:24:11+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-01T20:24:11+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-01T20:24:11.440011+09:00", "mtime_epoch": 1782905051.4400108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153294}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:24:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 903.004, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 164, "pushed_frames": 164, "read_frames": 164, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.474, "pushed": 5.474, "read": 5.474}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.098, "count": 163, "max_ms": 8.412, "p95_ms": 6.96}, "gstreamer_push": {"avg_ms": 0.996, "count": 164, "max_ms": 2.974, "p95_ms": 1.711}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.391, "count": 163, "max_ms": 70.219, "p95_ms": 67.241}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.868, "count": 163, "max_ms": 93.392, "p95_ms": 90.072}, "yolo11_640_fire_11class": {"avg_ms": 58.199, "count": 163, "max_ms": 75.497, "p95_ms": 72.892}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.019, "count": 163, "max_ms": 89.113, "p95_ms": 86.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 174.071, "count": 164, "max_ms": 220.958, "p95_ms": 196.163}, "read_interval": {"avg_ms": 183.589, "count": 164, "max_ms": 229.749, "p95_ms": 205.003}, "triton_wait": {"avg_ms": 69.881, "count": 163, "max_ms": 93.405, "p95_ms": 90.085}, "visualize": {"avg_ms": 7.703, "count": 163, "max_ms": 10.905, "p95_ms": 8.816}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 682}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4071.383, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:24:11+09:00", "last_error_summary": "[h264 @ 0x55a8f8e88c40] error while decoding MB 4 30, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 154, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:24:11+09:00", "last_error_summary": "[h264 @ 0x55a8f8e88c40] error while decoding MB 4 30, bytestream -7", "last_exit_at": "2026-07-01T20:24:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.794, "last_started_at": "2026-07-01T20:22:01+09:00", "pid": 22000, "process": {"exists": false, "pid": 22000}, "restart_count": 171, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 172}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.936, "sample_count": 1406, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81846, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:24:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T20:24:11+09:00", "last_frame_read_age_sec": 0.164, "last_frame_read_at": "2026-07-01T20:24:11+09:00", "mjpeg": {"age_sec": 0.163, "exists": true, "mtime": "2026-07-01T20:24:11.440011+09:00", "mtime_epoch": 1782905051.4400108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153294}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:24:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 903.004, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 164, "pushed_frames": 164, "read_frames": 163, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.491, "pushed": 5.491, "read": 5.457}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.097, "count": 164, "max_ms": 8.412, "p95_ms": 6.96}, "gstreamer_push": {"avg_ms": 0.997, "count": 164, "max_ms": 2.974, "p95_ms": 1.711}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.381, "count": 164, "max_ms": 70.219, "p95_ms": 67.241}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.807, "count": 164, "max_ms": 93.392, "p95_ms": 90.072}, "yolo11_640_fire_11class": {"avg_ms": 58.16, "count": 164, "max_ms": 75.497, "p95_ms": 72.892}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.961, "count": 164, "max_ms": 89.113, "p95_ms": 86.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 173.831, "count": 164, "max_ms": 220.958, "p95_ms": 195.738}, "read_interval": {"avg_ms": 183.445, "count": 163, "max_ms": 229.749, "p95_ms": 204.894}, "triton_wait": {"avg_ms": 69.819, "count": 164, "max_ms": 93.405, "p95_ms": 90.085}, "visualize": {"avg_ms": 7.701, "count": 164, "max_ms": 10.905, "p95_ms": 8.816}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 682}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4071.5, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 154, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:24:11+09:00", "last_error_summary": "[h264 @ 0x55a8f8e88c40] error while decoding MB 4 30, bytestream -7", "last_exit_at": "2026-07-01T20:24:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.798, "last_started_at": "2026-07-01T20:22:01+09:00", "pid": 22000, "process": {"exists": false, "pid": 22000}, "restart_count": 171, "restart_count_per_hour": 31, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 172}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.936, "sample_count": 1406, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81846, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:24:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T20:24:11+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-01T20:24:11+09:00", "mjpeg": {"age_sec": 0.166, "exists": true, "mtime": "2026-07-01T20:24:11.440011+09:00", "mtime_epoch": 1782905051.4400108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 153294}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:24:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 903.004, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 164, "pushed_frames": 164, "read_frames": 163, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.49, "pushed": 5.49, "read": 5.456}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.097, "count": 164, "max_ms": 8.412, "p95_ms": 6.96}, "gstreamer_push": {"avg_ms": 0.997, "count": 164, "max_ms": 2.974, "p95_ms": 1.711}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.381, "count": 164, "max_ms": 70.219, "p95_ms": 67.241}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.807, "count": 164, "max_ms": 93.392, "p95_ms": 90.072}, "yolo11_640_fire_11class": {"avg_ms": 58.16, "count": 164, "max_ms": 75.497, "p95_ms": 72.892}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.961, "count": 164, "max_ms": 89.113, "p95_ms": 86.791}}, "truncated_count": 0}, "processing_total": {"avg_ms": 173.831, "count": 164, "max_ms": 220.958, "p95_ms": 195.738}, "read_interval": {"avg_ms": 183.445, "count": 163, "max_ms": 229.749, "p95_ms": 204.894}, "triton_wait": {"avg_ms": 69.819, "count": 164, "max_ms": 93.405, "p95_ms": 90.085}, "visualize": {"avg_ms": 7.701, "count": 164, "max_ms": 10.905, "p95_ms": 8.816}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 682}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4179.523, "limit_mb": 8796093022207.996}, "detail": {"pid": 22107, "restart_count": 172, "start_count": 173}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 148, "h264_decode_error_count": 154, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:24:11+09:00", "last_error_summary": "[h264 @ 0x55a8f8e88c40] error while decoding MB 4 30, bytestream -7", "last_exit_at": "2026-07-01T20:24:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:24:13+09:00", "pid": 22107, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.082, "name": "ffmpeg", "pid": 22107, "rss_mb": 142.082, "state": "S (sleeping)", "threads": 106}, "restart_count": 172, "restart_count_per_hour": 32, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 173}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.761, "sample_count": 1390, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81846, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:24:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.024, "last_frame_pushed_at": "2026-07-01T20:24:11+09:00", "last_frame_read_age_sec": 4.185, "last_frame_read_at": "2026-07-01T20:24:11+09:00", "mjpeg": {"age_sec": 4.012, "exists": true, "mtime": "2026-07-01T20:24:11.612014+09:00", "mtime_epoch": 1782905051.6120143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 152458}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:24:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 905.113, "name": "celery", "pid": 237, "rss_mb": 903.004, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 141, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.749, "pushed": 4.749, "read": 4.715}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.003, "count": 142, "max_ms": 8.412, "p95_ms": 6.307}, "gstreamer_push": {"avg_ms": 1.0, "count": 142, "max_ms": 2.974, "p95_ms": 1.711}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.996, "count": 142, "max_ms": 70.219, "p95_ms": 67.352}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.534, "count": 142, "max_ms": 93.392, "p95_ms": 90.268}, "yolo11_640_fire_11class": {"avg_ms": 58.568, "count": 142, "max_ms": 75.497, "p95_ms": 73.129}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.756, "count": 142, "max_ms": 89.113, "p95_ms": 86.824}}, "truncated_count": 0}, "processing_total": {"avg_ms": 173.991, "count": 142, "max_ms": 220.958, "p95_ms": 195.183}, "read_interval": {"avg_ms": 183.577, "count": 141, "max_ms": 229.749, "p95_ms": 204.894}, "triton_wait": {"avg_ms": 70.547, "count": 142, "max_ms": 93.405, "p95_ms": 90.281}, "visualize": {"avg_ms": 7.632, "count": 142, "max_ms": 10.905, "p95_ms": 7.874}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4067.75, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x564872a34900] error while decoding MB 8 26, bytestream -34 [in#0/rtsp @ 0x5648728f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648728f7e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 155, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:26:44+09:00", "last_error_summary": "[h264 @ 0x564872a34900] error while decoding MB 8 26, bytestream -34 [in#0/rtsp @ 0x5648728f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648728f7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T20:24:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.723, "last_started_at": "2026-07-01T20:24:13+09:00", "pid": 22107, "process": {"exists": false, "pid": 22107}, "restart_count": 172, "restart_count_per_hour": 30, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 173}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 196.24, "sample_count": 1532, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 82604, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:26:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-01T20:26:44+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-01T20:26:44+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-01T20:26:44.295054+09:00", "mtime_epoch": 1782905204.2950544, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151117}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:26:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 903.789, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 66, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.211, "pushed": 2.211, "read": 2.211}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.858, "count": 66, "max_ms": 9.316, "p95_ms": 7.098}, "gstreamer_push": {"avg_ms": 1.028, "count": 66, "max_ms": 1.925, "p95_ms": 1.176}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.568, "count": 66, "max_ms": 75.746, "p95_ms": 68.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.143, "count": 66, "max_ms": 93.436, "p95_ms": 90.705}, "yolo11_640_fire_11class": {"avg_ms": 59.379, "count": 66, "max_ms": 79.947, "p95_ms": 77.431}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.825, "count": 66, "max_ms": 92.362, "p95_ms": 89.121}}, "truncated_count": 0}, "processing_total": {"avg_ms": 164.651, "count": 66, "max_ms": 201.069, "p95_ms": 194.891}, "read_interval": {"avg_ms": 453.898, "count": 66, "max_ms": 18544.585, "p95_ms": 210.618}, "triton_wait": {"avg_ms": 69.305, "count": 66, "max_ms": 93.448, "p95_ms": 91.199}, "visualize": {"avg_ms": 7.415, "count": 66, "max_ms": 12.403, "p95_ms": 9.664}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.851, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4069.238, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:26:44+09:00", "last_error_summary": "[h264 @ 0x564872a34900] error while decoding MB 8 26, bytestream -34 [in#0/rtsp @ 0x5648728f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648728f7e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 155, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:26:44+09:00", "last_error_summary": "[h264 @ 0x564872a34900] error while decoding MB 8 26, bytestream -34 [in#0/rtsp @ 0x5648728f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648728f7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T20:26:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.83, "last_started_at": "2026-07-01T20:24:13+09:00", "pid": 22107, "process": {"exists": false, "pid": 22107}, "restart_count": 172, "restart_count_per_hour": 30, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 173}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 196.204, "sample_count": 1532, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 82605, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:26:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T20:26:44+09:00", "last_frame_read_age_sec": 0.168, "last_frame_read_at": "2026-07-01T20:26:44+09:00", "mjpeg": {"age_sec": 0.158, "exists": true, "mtime": "2026-07-01T20:26:44.295054+09:00", "mtime_epoch": 1782905204.2950544, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151117}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:26:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 903.789, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.237, "pushed": 2.237, "read": 2.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.855, "count": 66, "max_ms": 9.316, "p95_ms": 7.098}, "gstreamer_push": {"avg_ms": 1.03, "count": 67, "max_ms": 1.925, "p95_ms": 1.176}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.772, "count": 66, "max_ms": 75.746, "p95_ms": 68.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.979, "count": 66, "max_ms": 93.436, "p95_ms": 90.705}, "yolo11_640_fire_11class": {"avg_ms": 59.409, "count": 66, "max_ms": 79.947, "p95_ms": 77.431}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.71, "count": 66, "max_ms": 92.362, "p95_ms": 89.121}}, "truncated_count": 0}, "processing_total": {"avg_ms": 164.606, "count": 67, "max_ms": 201.069, "p95_ms": 194.891}, "read_interval": {"avg_ms": 453.898, "count": 66, "max_ms": 18544.585, "p95_ms": 210.618}, "triton_wait": {"avg_ms": 69.142, "count": 66, "max_ms": 93.448, "p95_ms": 91.199}, "visualize": {"avg_ms": 7.411, "count": 66, "max_ms": 12.403, "p95_ms": 9.664}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4069.23, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 155, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:26:44+09:00", "last_error_summary": "[h264 @ 0x564872a34900] error while decoding MB 8 26, bytestream -34 [in#0/rtsp @ 0x5648728f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648728f7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T20:26:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.833, "last_started_at": "2026-07-01T20:24:13+09:00", "pid": 22107, "process": {"exists": false, "pid": 22107}, "restart_count": 172, "restart_count_per_hour": 30, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 173}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 196.204, "sample_count": 1532, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 82605, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:26:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T20:26:44+09:00", "last_frame_read_age_sec": 0.172, "last_frame_read_at": "2026-07-01T20:26:44+09:00", "mjpeg": {"age_sec": 0.161, "exists": true, "mtime": "2026-07-01T20:26:44.295054+09:00", "mtime_epoch": 1782905204.2950544, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151117}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:26:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 903.789, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.236, "pushed": 2.236, "read": 2.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.855, "count": 66, "max_ms": 9.316, "p95_ms": 7.098}, "gstreamer_push": {"avg_ms": 1.03, "count": 67, "max_ms": 1.925, "p95_ms": 1.176}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.772, "count": 66, "max_ms": 75.746, "p95_ms": 68.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.979, "count": 66, "max_ms": 93.436, "p95_ms": 90.705}, "yolo11_640_fire_11class": {"avg_ms": 59.409, "count": 66, "max_ms": 79.947, "p95_ms": 77.431}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.71, "count": 66, "max_ms": 92.362, "p95_ms": 89.121}}, "truncated_count": 0}, "processing_total": {"avg_ms": 164.606, "count": 67, "max_ms": 201.069, "p95_ms": 194.891}, "read_interval": {"avg_ms": 453.898, "count": 66, "max_ms": 18544.585, "p95_ms": 210.618}, "triton_wait": {"avg_ms": 69.142, "count": 66, "max_ms": 93.448, "p95_ms": 91.199}, "visualize": {"avg_ms": 7.411, "count": 66, "max_ms": 12.403, "p95_ms": 9.664}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 688}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4180.105, "limit_mb": 8796093022207.996}, "detail": {"pid": 22214, "restart_count": 173, "start_count": 174}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 149, "h264_decode_error_count": 155, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:26:44+09:00", "last_error_summary": "[h264 @ 0x564872a34900] error while decoding MB 8 26, bytestream -34 [in#0/rtsp @ 0x5648728f7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5648728f7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T20:26:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:26:46+09:00", "pid": 22214, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.176, "name": "ffmpeg", "pid": 22214, "rss_mb": 139.176, "state": "S (sleeping)", "threads": 106}, "restart_count": 173, "restart_count_per_hour": 31, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 174}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 195.17, "sample_count": 1516, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 82605, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:26:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.021, "last_frame_pushed_at": "2026-07-01T20:26:44+09:00", "last_frame_read_age_sec": 4.185, "last_frame_read_at": "2026-07-01T20:26:44+09:00", "mjpeg": {"age_sec": 4.002, "exists": true, "mtime": "2026-07-01T20:26:44.467058+09:00", "mtime_epoch": 1782905204.4670577, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151107}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:26:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 903.789, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 45, "pushed_frames": 45, "read_frames": 44, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.503, "pushed": 1.503, "read": 1.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.729, "count": 44, "max_ms": 7.601, "p95_ms": 7.098}, "gstreamer_push": {"avg_ms": 1.028, "count": 45, "max_ms": 1.788, "p95_ms": 1.14}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.31, "count": 44, "max_ms": 68.86, "p95_ms": 60.186}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.376, "count": 44, "max_ms": 93.436, "p95_ms": 81.065}, "yolo11_640_fire_11class": {"avg_ms": 57.67, "count": 44, "max_ms": 79.947, "p95_ms": 76.473}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.428, "count": 44, "max_ms": 91.532, "p95_ms": 81.292}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.796, "count": 45, "max_ms": 201.069, "p95_ms": 183.391}, "read_interval": {"avg_ms": 589.317, "count": 44, "max_ms": 18544.585, "p95_ms": 202.934}, "triton_wait": {"avg_ms": 67.502, "count": 44, "max_ms": 93.448, "p95_ms": 83.317}, "visualize": {"avg_ms": 7.232, "count": 44, "max_ms": 9.664, "p95_ms": 9.032}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4068.82, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55795af2f800] error while decoding MB 76 2, bytestream -26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 150, "h264_decode_error_count": 156, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:28:58+09:00", "last_error_summary": "[h264 @ 0x55795af2f800] error while decoding MB 76 2, bytestream -26", "last_exit_at": "2026-07-01T20:26:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.847, "last_started_at": "2026-07-01T20:26:46+09:00", "pid": 22214, "process": {"exists": false, "pid": 22214}, "restart_count": 173, "restart_count_per_hour": 30, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 174}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.596, "sample_count": 1590, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83364, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:28:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T20:28:58+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T20:28:58+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-01T20:28:58.269682+09:00", "mtime_epoch": 1782905338.2696824, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151083}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:28:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 901.805, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.341, "pushed": 6.341, "read": 6.341}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.189, "count": 189, "max_ms": 6.896, "p95_ms": 6.259}, "gstreamer_push": {"avg_ms": 1.133, "count": 190, "max_ms": 4.26, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.002, "count": 189, "max_ms": 73.477, "p95_ms": 66.596}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.737, "count": 189, "max_ms": 98.005, "p95_ms": 92.038}, "yolo11_640_fire_11class": {"avg_ms": 55.33, "count": 189, "max_ms": 76.488, "p95_ms": 71.651}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.082, "count": 189, "max_ms": 92.459, "p95_ms": 90.348}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.956, "count": 190, "max_ms": 213.087, "p95_ms": 192.303}, "read_interval": {"avg_ms": 158.28, "count": 190, "max_ms": 224.266, "p95_ms": 202.025}, "triton_wait": {"avg_ms": 69.816, "count": 189, "max_ms": 98.018, "p95_ms": 92.051}, "visualize": {"avg_ms": 6.31, "count": 189, "max_ms": 8.886, "p95_ms": 8.148}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4068.062, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:28:58+09:00", "last_error_summary": "[h264 @ 0x55795af2f800] error while decoding MB 76 2, bytestream -26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 150, "h264_decode_error_count": 156, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:28:58+09:00", "last_error_summary": "[h264 @ 0x55795af2f800] error while decoding MB 76 2, bytestream -26", "last_exit_at": "2026-07-01T20:28:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.94, "last_started_at": "2026-07-01T20:26:46+09:00", "pid": 22214, "process": {"exists": false, "pid": 22214}, "restart_count": 173, "restart_count_per_hour": 30, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 174}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.571, "sample_count": 1591, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83365, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:28:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T20:28:58+09:00", "last_frame_read_age_sec": 0.147, "last_frame_read_at": "2026-07-01T20:28:58+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-01T20:28:58.269682+09:00", "mtime_epoch": 1782905338.2696824, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151083}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:28:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 901.805, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.353, "pushed": 6.353, "read": 6.32}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.19, "count": 190, "max_ms": 6.896, "p95_ms": 6.259}, "gstreamer_push": {"avg_ms": 1.134, "count": 190, "max_ms": 4.26, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.971, "count": 190, "max_ms": 73.477, "p95_ms": 66.596}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.678, "count": 190, "max_ms": 98.005, "p95_ms": 92.038}, "yolo11_640_fire_11class": {"avg_ms": 55.275, "count": 190, "max_ms": 76.488, "p95_ms": 71.651}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.033, "count": 190, "max_ms": 92.459, "p95_ms": 90.348}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.843, "count": 190, "max_ms": 213.087, "p95_ms": 192.303}, "read_interval": {"avg_ms": 158.207, "count": 189, "max_ms": 224.266, "p95_ms": 202.025}, "triton_wait": {"avg_ms": 69.757, "count": 190, "max_ms": 98.018, "p95_ms": 92.051}, "visualize": {"avg_ms": 6.316, "count": 190, "max_ms": 8.886, "p95_ms": 8.148}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4068.008, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 150, "h264_decode_error_count": 156, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:28:58+09:00", "last_error_summary": "[h264 @ 0x55795af2f800] error while decoding MB 76 2, bytestream -26", "last_exit_at": "2026-07-01T20:28:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.944, "last_started_at": "2026-07-01T20:26:46+09:00", "pid": 22214, "process": {"exists": false, "pid": 22214}, "restart_count": 173, "restart_count_per_hour": 30, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 174}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.571, "sample_count": 1591, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83365, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:28:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T20:28:58+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-01T20:28:58+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-01T20:28:58.269682+09:00", "mtime_epoch": 1782905338.2696824, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 151083}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:28:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 901.805, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.352, "pushed": 6.352, "read": 6.319}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.19, "count": 190, "max_ms": 6.896, "p95_ms": 6.259}, "gstreamer_push": {"avg_ms": 1.134, "count": 190, "max_ms": 4.26, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.971, "count": 190, "max_ms": 73.477, "p95_ms": 66.596}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.678, "count": 190, "max_ms": 98.005, "p95_ms": 92.038}, "yolo11_640_fire_11class": {"avg_ms": 55.275, "count": 190, "max_ms": 76.488, "p95_ms": 71.651}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.033, "count": 190, "max_ms": 92.459, "p95_ms": 90.348}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.843, "count": 190, "max_ms": 213.087, "p95_ms": 192.303}, "read_interval": {"avg_ms": 158.207, "count": 189, "max_ms": 224.266, "p95_ms": 202.025}, "triton_wait": {"avg_ms": 69.757, "count": 190, "max_ms": 98.018, "p95_ms": 92.051}, "visualize": {"avg_ms": 6.316, "count": 190, "max_ms": 8.886, "p95_ms": 8.148}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 694}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4178.508, "limit_mb": 8796093022207.996}, "detail": {"pid": 22321, "restart_count": 174, "start_count": 175}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 150, "h264_decode_error_count": 156, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:28:58+09:00", "last_error_summary": "[h264 @ 0x55795af2f800] error while decoding MB 76 2, bytestream -26", "last_exit_at": "2026-07-01T20:29:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:29:01+09:00", "pid": 22321, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.055, "name": "ffmpeg", "pid": 22321, "rss_mb": 136.055, "state": "S (sleeping)", "threads": 106}, "restart_count": 174, "restart_count_per_hour": 31, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 175}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 193.543, "sample_count": 1566, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83365, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:28:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.849, "last_frame_pushed_at": "2026-07-01T20:28:58+09:00", "last_frame_read_age_sec": 4.992, "last_frame_read_at": "2026-07-01T20:28:58+09:00", "mjpeg": {"age_sec": 4.832, "exists": true, "mtime": "2026-07-01T20:28:58.421685+09:00", "mtime_epoch": 1782905338.4216852, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149192}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:29:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 901.805, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 159, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.349, "pushed": 5.349, "read": 5.316}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.217, "count": 160, "max_ms": 6.896, "p95_ms": 6.259}, "gstreamer_push": {"avg_ms": 1.15, "count": 160, "max_ms": 4.26, "p95_ms": 1.791}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.532, "count": 160, "max_ms": 73.477, "p95_ms": 66.121}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.68, "count": 160, "max_ms": 98.005, "p95_ms": 91.783}, "yolo11_640_fire_11class": {"avg_ms": 54.51, "count": 160, "max_ms": 76.488, "p95_ms": 71.44}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.031, "count": 160, "max_ms": 92.459, "p95_ms": 89.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.256, "count": 160, "max_ms": 213.087, "p95_ms": 185.582}, "read_interval": {"avg_ms": 157.67, "count": 159, "max_ms": 224.266, "p95_ms": 196.824}, "triton_wait": {"avg_ms": 68.758, "count": 160, "max_ms": 98.018, "p95_ms": 91.796}, "visualize": {"avg_ms": 6.338, "count": 160, "max_ms": 8.886, "p95_ms": 8.17}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 702}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4072.652, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564ecb1e92c0] error while decoding MB 32 0, bytestream -18", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 151, "h264_decode_error_count": 157, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:31:34+09:00", "last_error_summary": "[h264 @ 0x564ecb1e92c0] error while decoding MB 32 0, bytestream -18", "last_exit_at": "2026-07-01T20:29:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.717, "last_started_at": "2026-07-01T20:29:01+09:00", "pid": 22321, "process": {"exists": false, "pid": 22321}, "restart_count": 174, "restart_count_per_hour": 30, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 175}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 190.557, "sample_count": 1739, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 84336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:31:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-01T20:31:34+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-01T20:31:34+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-01T20:31:34.916720+09:00", "mtime_epoch": 1782905494.9167197, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149074}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:31:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 902.137, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.294, "pushed": 6.294, "read": 6.294}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.27, "count": 188, "max_ms": 7.75, "p95_ms": 6.024}, "gstreamer_push": {"avg_ms": 1.134, "count": 188, "max_ms": 3.015, "p95_ms": 1.861}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.057, "count": 188, "max_ms": 71.446, "p95_ms": 63.417}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.83, "count": 188, "max_ms": 93.41, "p95_ms": 89.597}, "yolo11_640_fire_11class": {"avg_ms": 54.603, "count": 188, "max_ms": 74.872, "p95_ms": 70.192}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.772, "count": 188, "max_ms": 90.671, "p95_ms": 86.405}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.107, "count": 188, "max_ms": 188.65, "p95_ms": 172.794}, "read_interval": {"avg_ms": 159.445, "count": 188, "max_ms": 201.99, "p95_ms": 184.093}, "triton_wait": {"avg_ms": 68.052, "count": 188, "max_ms": 93.423, "p95_ms": 89.61}, "visualize": {"avg_ms": 6.58, "count": 188, "max_ms": 10.042, "p95_ms": 7.895}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 702}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4072.066, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:31:34+09:00", "last_error_summary": "[h264 @ 0x564ecb1e92c0] error while decoding MB 32 0, bytestream -18", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 151, "h264_decode_error_count": 157, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:31:34+09:00", "last_error_summary": "[h264 @ 0x564ecb1e92c0] error while decoding MB 32 0, bytestream -18", "last_exit_at": "2026-07-01T20:31:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.816, "last_started_at": "2026-07-01T20:29:01+09:00", "pid": 22321, "process": {"exists": false, "pid": 22321}, "restart_count": 174, "restart_count_per_hour": 30, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 175}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 190.557, "sample_count": 1740, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 84337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:31:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T20:31:35+09:00", "last_frame_read_age_sec": 0.159, "last_frame_read_at": "2026-07-01T20:31:34+09:00", "mjpeg": {"age_sec": 0.152, "exists": true, "mtime": "2026-07-01T20:31:34.916720+09:00", "mtime_epoch": 1782905494.9167197, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149074}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:31:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 902.137, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.307, "pushed": 6.307, "read": 6.273}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.29, "count": 188, "max_ms": 8.989, "p95_ms": 6.074}, "gstreamer_push": {"avg_ms": 1.134, "count": 189, "max_ms": 3.015, "p95_ms": 1.861}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.964, "count": 188, "max_ms": 70.589, "p95_ms": 63.042}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.709, "count": 188, "max_ms": 93.41, "p95_ms": 89.597}, "yolo11_640_fire_11class": {"avg_ms": 54.511, "count": 188, "max_ms": 74.872, "p95_ms": 70.042}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.64, "count": 188, "max_ms": 90.671, "p95_ms": 86.393}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.133, "count": 189, "max_ms": 188.65, "p95_ms": 172.794}, "read_interval": {"avg_ms": 159.445, "count": 188, "max_ms": 201.99, "p95_ms": 184.093}, "triton_wait": {"avg_ms": 67.925, "count": 188, "max_ms": 93.423, "p95_ms": 89.61}, "visualize": {"avg_ms": 6.582, "count": 188, "max_ms": 10.042, "p95_ms": 7.895}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 702}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4072.059, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 151, "h264_decode_error_count": 157, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:31:34+09:00", "last_error_summary": "[h264 @ 0x564ecb1e92c0] error while decoding MB 32 0, bytestream -18", "last_exit_at": "2026-07-01T20:31:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.82, "last_started_at": "2026-07-01T20:29:01+09:00", "pid": 22321, "process": {"exists": false, "pid": 22321}, "restart_count": 174, "restart_count_per_hour": 30, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 175}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 190.557, "sample_count": 1740, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 84337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:31:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T20:31:35+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-01T20:31:34+09:00", "mjpeg": {"age_sec": 0.156, "exists": true, "mtime": "2026-07-01T20:31:34.916720+09:00", "mtime_epoch": 1782905494.9167197, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149074}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:31:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 902.137, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.306, "pushed": 6.306, "read": 6.273}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.29, "count": 188, "max_ms": 8.989, "p95_ms": 6.074}, "gstreamer_push": {"avg_ms": 1.134, "count": 189, "max_ms": 3.015, "p95_ms": 1.861}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.964, "count": 188, "max_ms": 70.589, "p95_ms": 63.042}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.709, "count": 188, "max_ms": 93.41, "p95_ms": 89.597}, "yolo11_640_fire_11class": {"avg_ms": 54.511, "count": 188, "max_ms": 74.872, "p95_ms": 70.042}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.64, "count": 188, "max_ms": 90.671, "p95_ms": 86.393}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.133, "count": 189, "max_ms": 188.65, "p95_ms": 172.794}, "read_interval": {"avg_ms": 159.445, "count": 188, "max_ms": 201.99, "p95_ms": 184.093}, "triton_wait": {"avg_ms": 67.925, "count": 188, "max_ms": 93.423, "p95_ms": 89.61}, "visualize": {"avg_ms": 6.582, "count": 188, "max_ms": 10.042, "p95_ms": 7.895}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 702}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4182.434, "limit_mb": 8796093022207.996}, "detail": {"pid": 22428, "restart_count": 175, "start_count": 176}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 151, "h264_decode_error_count": 157, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:31:34+09:00", "last_error_summary": "[h264 @ 0x564ecb1e92c0] error while decoding MB 32 0, bytestream -18", "last_exit_at": "2026-07-01T20:31:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:31:37+09:00", "pid": 22428, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.219, "name": "ffmpeg", "pid": 22428, "rss_mb": 139.219, "state": "S (sleeping)", "threads": 106}, "restart_count": 175, "restart_count_per_hour": 31, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 176}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 190.557, "sample_count": 1740, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 84337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:31:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.646, "last_frame_pushed_at": "2026-07-01T20:31:35+09:00", "last_frame_read_age_sec": 4.801, "last_frame_read_at": "2026-07-01T20:31:34+09:00", "mjpeg": {"age_sec": 4.63, "exists": true, "mtime": "2026-07-01T20:31:35.080723+09:00", "mtime_epoch": 1782905495.0807228, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149072}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:31:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 902.137, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 159, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.342, "pushed": 5.342, "read": 5.308}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.324, "count": 160, "max_ms": 8.989, "p95_ms": 6.129}, "gstreamer_push": {"avg_ms": 1.14, "count": 160, "max_ms": 3.015, "p95_ms": 1.861}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.01, "count": 160, "max_ms": 70.589, "p95_ms": 62.764}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.618, "count": 160, "max_ms": 93.41, "p95_ms": 89.717}, "yolo11_640_fire_11class": {"avg_ms": 54.544, "count": 160, "max_ms": 74.872, "p95_ms": 69.927}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.591, "count": 160, "max_ms": 90.671, "p95_ms": 86.405}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.69, "count": 160, "max_ms": 188.65, "p95_ms": 172.692}, "read_interval": {"avg_ms": 158.976, "count": 159, "max_ms": 201.99, "p95_ms": 184.093}, "triton_wait": {"avg_ms": 67.87, "count": 160, "max_ms": 93.423, "p95_ms": 89.729}, "visualize": {"avg_ms": 6.562, "count": 160, "max_ms": 9.26, "p95_ms": 7.886}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 710}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4069.133, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556162a9b500] error while decoding MB 77 21, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 152, "h264_decode_error_count": 158, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:34:10+09:00", "last_error_summary": "[h264 @ 0x556162a9b500] error while decoding MB 77 21, bytestream -9", "last_exit_at": "2026-07-01T20:31:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.672, "last_started_at": "2026-07-01T20:31:37+09:00", "pid": 22428, "process": {"exists": false, "pid": 22428}, "restart_count": 175, "restart_count_per_hour": 29, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 176}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.632, "sample_count": 1889, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85300, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:34:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-01T20:34:10+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T20:34:10+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T20:34:10.335705+09:00", "mtime_epoch": 1782905650.3357053, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 137689}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:34:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 899.719, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 194, "pushed_frames": 194, "read_frames": 194, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.469, "pushed": 6.469, "read": 6.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.096, "count": 193, "max_ms": 7.778, "p95_ms": 5.738}, "gstreamer_push": {"avg_ms": 1.136, "count": 194, "max_ms": 3.617, "p95_ms": 1.233}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.602, "count": 193, "max_ms": 73.829, "p95_ms": 67.586}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.982, "count": 193, "max_ms": 91.265, "p95_ms": 86.288}, "yolo11_640_fire_11class": {"avg_ms": 55.139, "count": 193, "max_ms": 75.979, "p95_ms": 70.459}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.123, "count": 193, "max_ms": 88.676, "p95_ms": 84.718}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.42, "count": 194, "max_ms": 184.174, "p95_ms": 165.219}, "read_interval": {"avg_ms": 155.056, "count": 194, "max_ms": 196.078, "p95_ms": 177.076}, "triton_wait": {"avg_ms": 66.307, "count": 193, "max_ms": 91.277, "p95_ms": 86.488}, "visualize": {"avg_ms": 6.403, "count": 193, "max_ms": 11.236, "p95_ms": 7.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 710}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4068.543, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:34:10+09:00", "last_error_summary": "[h264 @ 0x556162a9b500] error while decoding MB 77 21, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 152, "h264_decode_error_count": 158, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:34:10+09:00", "last_error_summary": "[h264 @ 0x556162a9b500] error while decoding MB 77 21, bytestream -9", "last_exit_at": "2026-07-01T20:34:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.769, "last_started_at": "2026-07-01T20:31:37+09:00", "pid": 22428, "process": {"exists": false, "pid": 22428}, "restart_count": 175, "restart_count_per_hour": 29, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 176}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.632, "sample_count": 1890, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85301, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:34:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T20:34:10+09:00", "last_frame_read_age_sec": 0.149, "last_frame_read_at": "2026-07-01T20:34:10+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-01T20:34:10.335705+09:00", "mtime_epoch": 1782905650.3357053, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 137689}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:34:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 899.719, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 194, "pushed_frames": 194, "read_frames": 193, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.481, "pushed": 6.481, "read": 6.447}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.102, "count": 193, "max_ms": 7.778, "p95_ms": 5.795}, "gstreamer_push": {"avg_ms": 1.135, "count": 194, "max_ms": 3.617, "p95_ms": 1.233}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.623, "count": 193, "max_ms": 73.829, "p95_ms": 67.586}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.0, "count": 193, "max_ms": 91.265, "p95_ms": 86.288}, "yolo11_640_fire_11class": {"avg_ms": 55.207, "count": 193, "max_ms": 75.979, "p95_ms": 70.459}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.153, "count": 193, "max_ms": 88.676, "p95_ms": 84.718}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.422, "count": 194, "max_ms": 184.174, "p95_ms": 165.219}, "read_interval": {"avg_ms": 155.057, "count": 193, "max_ms": 196.078, "p95_ms": 177.076}, "triton_wait": {"avg_ms": 66.325, "count": 193, "max_ms": 91.277, "p95_ms": 86.488}, "visualize": {"avg_ms": 6.403, "count": 193, "max_ms": 11.236, "p95_ms": 7.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 710}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4068.535, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 152, "h264_decode_error_count": 158, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:34:10+09:00", "last_error_summary": "[h264 @ 0x556162a9b500] error while decoding MB 77 21, bytestream -9", "last_exit_at": "2026-07-01T20:34:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 152.773, "last_started_at": "2026-07-01T20:31:37+09:00", "pid": 22428, "process": {"exists": false, "pid": 22428}, "restart_count": 175, "restart_count_per_hour": 29, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 176}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.632, "sample_count": 1890, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85301, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:34:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T20:34:10+09:00", "last_frame_read_age_sec": 0.154, "last_frame_read_at": "2026-07-01T20:34:10+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-01T20:34:10.335705+09:00", "mtime_epoch": 1782905650.3357053, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 137689}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:34:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 899.719, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 194, "pushed_frames": 194, "read_frames": 193, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.48, "pushed": 6.48, "read": 6.447}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.102, "count": 193, "max_ms": 7.778, "p95_ms": 5.795}, "gstreamer_push": {"avg_ms": 1.135, "count": 194, "max_ms": 3.617, "p95_ms": 1.233}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.623, "count": 193, "max_ms": 73.829, "p95_ms": 67.586}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.0, "count": 193, "max_ms": 91.265, "p95_ms": 86.288}, "yolo11_640_fire_11class": {"avg_ms": 55.207, "count": 193, "max_ms": 75.979, "p95_ms": 70.459}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.153, "count": 193, "max_ms": 88.676, "p95_ms": 84.718}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.422, "count": 194, "max_ms": 184.174, "p95_ms": 165.219}, "read_interval": {"avg_ms": 155.057, "count": 193, "max_ms": 196.078, "p95_ms": 177.076}, "triton_wait": {"avg_ms": 66.325, "count": 193, "max_ms": 91.277, "p95_ms": 86.488}, "visualize": {"avg_ms": 6.403, "count": 193, "max_ms": 11.236, "p95_ms": 7.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 710}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4178.484, "limit_mb": 8796093022207.996}, "detail": {"pid": 22535, "restart_count": 176, "start_count": 177}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 152, "h264_decode_error_count": 158, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:34:10+09:00", "last_error_summary": "[h264 @ 0x556162a9b500] error while decoding MB 77 21, bytestream -9", "last_exit_at": "2026-07-01T20:34:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:34:13+09:00", "pid": 22535, "process": {"exists": true, "fd_count": 5, "hwm_mb": 140.996, "name": "ffmpeg", "pid": 22535, "rss_mb": 140.996, "state": "S (sleeping)", "threads": 106}, "restart_count": 176, "restart_count_per_hour": 30, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 177}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.507, "sample_count": 1861, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85301, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:34:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.547, "last_frame_pushed_at": "2026-07-01T20:34:10+09:00", "last_frame_read_age_sec": 4.692, "last_frame_read_at": "2026-07-01T20:34:10+09:00", "mjpeg": {"age_sec": 4.529, "exists": true, "mtime": "2026-07-01T20:34:10.491708+09:00", "mtime_epoch": 1782905650.4917083, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136244}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:34:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 899.719, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 165, "pushed_frames": 165, "read_frames": 164, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.509, "pushed": 5.509, "read": 5.475}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.103, "count": 164, "max_ms": 7.778, "p95_ms": 5.795}, "gstreamer_push": {"avg_ms": 1.137, "count": 165, "max_ms": 3.617, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.337, "count": 164, "max_ms": 73.829, "p95_ms": 67.586}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.21, "count": 164, "max_ms": 91.265, "p95_ms": 85.007}, "yolo11_640_fire_11class": {"avg_ms": 55.627, "count": 164, "max_ms": 75.979, "p95_ms": 70.293}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.798, "count": 164, "max_ms": 88.676, "p95_ms": 84.718}}, "truncated_count": 0}, "processing_total": {"avg_ms": 143.259, "count": 165, "max_ms": 184.174, "p95_ms": 164.137}, "read_interval": {"avg_ms": 154.846, "count": 164, "max_ms": 196.078, "p95_ms": 176.902}, "triton_wait": {"avg_ms": 66.59, "count": 164, "max_ms": 91.277, "p95_ms": 86.042}, "visualize": {"avg_ms": 6.39, "count": 164, "max_ms": 11.236, "p95_ms": 7.333}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 720}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4075.23, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b8807a2540] error while decoding MB 41 41, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:36:56+09:00", "last_error_summary": "[h264 @ 0x55b8807a2540] error while decoding MB 41 41, bytestream -5", "last_exit_at": "2026-07-01T20:34:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.741, "last_started_at": "2026-07-01T20:34:13+09:00", "pid": 22535, "process": {"exists": false, "pid": 22535}, "restart_count": 176, "restart_count_per_hour": 29, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 177}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.063, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 86414, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:36:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-01T20:36:56+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T20:36:56+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-01T20:36:56.715236+09:00", "mtime_epoch": 1782905816.715236, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 134418}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:36:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 900.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 220, "pushed_frames": 220, "read_frames": 220, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.341, "pushed": 7.341, "read": 7.341}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.359, "count": 219, "max_ms": 7.732, "p95_ms": 6.13}, "gstreamer_push": {"avg_ms": 1.293, "count": 220, "max_ms": 3.92, "p95_ms": 2.404}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.051, "count": 219, "max_ms": 75.233, "p95_ms": 66.135}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.67, "count": 219, "max_ms": 92.548, "p95_ms": 89.521}, "yolo11_640_fire_11class": {"avg_ms": 58.022, "count": 219, "max_ms": 78.66, "p95_ms": 73.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.303, "count": 219, "max_ms": 91.311, "p95_ms": 87.954}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.334, "count": 220, "max_ms": 171.087, "p95_ms": 161.439}, "read_interval": {"avg_ms": 136.513, "count": 220, "max_ms": 185.151, "p95_ms": 172.554}, "triton_wait": {"avg_ms": 68.831, "count": 219, "max_ms": 92.561, "p95_ms": 89.534}, "visualize": {"avg_ms": 5.533, "count": 219, "max_ms": 11.511, "p95_ms": 7.681}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 720}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4074.668, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:36:56+09:00", "last_error_summary": "[h264 @ 0x55b8807a2540] error while decoding MB 41 41, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:36:56+09:00", "last_error_summary": "[h264 @ 0x55b8807a2540] error while decoding MB 41 41, bytestream -5", "last_exit_at": "2026-07-01T20:36:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.81, "last_started_at": "2026-07-01T20:34:13+09:00", "pid": 22535, "process": {"exists": false, "pid": 22535}, "restart_count": 176, "restart_count_per_hour": 29, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 177}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.063, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 86415, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:36:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T20:36:56+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-01T20:36:56+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-01T20:36:56.715236+09:00", "mtime_epoch": 1782905816.715236, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 134418}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:36:56+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 900.293, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 220, "pushed_frames": 220, "read_frames": 219, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.365, "pushed": 7.365, "read": 7.331}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.359, "count": 220, "max_ms": 7.732, "p95_ms": 6.08}, "gstreamer_push": {"avg_ms": 1.292, "count": 220, "max_ms": 3.92, "p95_ms": 2.404}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.045, "count": 220, "max_ms": 75.233, "p95_ms": 65.245}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.645, "count": 220, "max_ms": 92.548, "p95_ms": 89.263}, "yolo11_640_fire_11class": {"avg_ms": 58.015, "count": 220, "max_ms": 78.66, "p95_ms": 73.832}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.28, "count": 220, "max_ms": 91.311, "p95_ms": 87.842}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.322, "count": 220, "max_ms": 171.087, "p95_ms": 161.439}, "read_interval": {"avg_ms": 136.546, "count": 219, "max_ms": 185.151, "p95_ms": 173.915}, "triton_wait": {"avg_ms": 68.805, "count": 220, "max_ms": 92.561, "p95_ms": 89.335}, "visualize": {"avg_ms": 5.531, "count": 220, "max_ms": 11.511, "p95_ms": 7.5}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 720}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4074.863, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:36:56+09:00", "last_error_summary": "[h264 @ 0x55b8807a2540] error while decoding MB 41 41, bytestream -5", "last_exit_at": "2026-07-01T20:36:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.816, "last_started_at": "2026-07-01T20:34:13+09:00", "pid": 22535, "process": {"exists": false, "pid": 22535}, "restart_count": 176, "restart_count_per_hour": 29, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 177}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.063, "sample_count": 1970, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 86415, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:36:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T20:36:56+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-01T20:36:56+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-01T20:36:56.715236+09:00", "mtime_epoch": 1782905816.715236, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 134418}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:36:56+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 900.293, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 220, "pushed_frames": 220, "read_frames": 219, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.363, "pushed": 7.363, "read": 7.33}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.359, "count": 220, "max_ms": 7.732, "p95_ms": 6.08}, "gstreamer_push": {"avg_ms": 1.292, "count": 220, "max_ms": 3.92, "p95_ms": 2.404}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.045, "count": 220, "max_ms": 75.233, "p95_ms": 65.245}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.645, "count": 220, "max_ms": 92.548, "p95_ms": 89.263}, "yolo11_640_fire_11class": {"avg_ms": 58.015, "count": 220, "max_ms": 78.66, "p95_ms": 73.832}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.28, "count": 220, "max_ms": 91.311, "p95_ms": 87.842}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.322, "count": 220, "max_ms": 171.087, "p95_ms": 161.439}, "read_interval": {"avg_ms": 136.546, "count": 219, "max_ms": 185.151, "p95_ms": 173.915}, "triton_wait": {"avg_ms": 68.805, "count": 220, "max_ms": 92.561, "p95_ms": 89.335}, "visualize": {"avg_ms": 5.531, "count": 220, "max_ms": 11.511, "p95_ms": 7.5}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 720}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4180.965, "limit_mb": 8796093022207.996}, "detail": {"pid": 22642, "restart_count": 177, "start_count": 178}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:36:56+09:00", "last_error_summary": "[h264 @ 0x55b8807a2540] error while decoding MB 41 41, bytestream -5", "last_exit_at": "2026-07-01T20:36:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:36:59+09:00", "pid": 22642, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.898, "name": "ffmpeg", "pid": 22642, "rss_mb": 136.898, "state": "S (sleeping)", "threads": 106}, "restart_count": 177, "restart_count_per_hour": 30, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 178}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 167.013, "sample_count": 1941, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 86415, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:36:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.568, "last_frame_pushed_at": "2026-07-01T20:36:56+09:00", "last_frame_read_age_sec": 4.684, "last_frame_read_at": "2026-07-01T20:36:56+09:00", "mjpeg": {"age_sec": 4.55, "exists": true, "mtime": "2026-07-01T20:36:56.843240+09:00", "mtime_epoch": 1782905816.8432403, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 134607}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:37:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 900.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 187, "pushed_frames": 188, "read_frames": 187, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.234, "pushed": 6.267, "read": 6.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.31, "count": 187, "max_ms": 7.732, "p95_ms": 6.13}, "gstreamer_push": {"avg_ms": 1.311, "count": 188, "max_ms": 3.92, "p95_ms": 2.476}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.943, "count": 187, "max_ms": 75.233, "p95_ms": 65.245}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.947, "count": 187, "max_ms": 92.548, "p95_ms": 89.521}, "yolo11_640_fire_11class": {"avg_ms": 58.156, "count": 187, "max_ms": 78.66, "p95_ms": 73.832}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.53, "count": 187, "max_ms": 91.311, "p95_ms": 87.954}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.688, "count": 187, "max_ms": 170.833, "p95_ms": 161.439}, "read_interval": {"avg_ms": 135.979, "count": 187, "max_ms": 185.151, "p95_ms": 172.554}, "triton_wait": {"avg_ms": 69.093, "count": 187, "max_ms": 92.561, "p95_ms": 89.534}, "visualize": {"avg_ms": 5.508, "count": 187, "max_ms": 11.069, "p95_ms": 7.681}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 732}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4074.895, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:36:56+09:00", "last_error_summary": "[h264 @ 0x55b8807a2540] error while decoding MB 41 41, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:36:56+09:00", "last_error_summary": "[h264 @ 0x55b8807a2540] error while decoding MB 41 41, bytestream -5", "last_exit_at": "2026-07-01T20:40:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 205.829, "last_started_at": "2026-07-01T20:36:59+09:00", "pid": 22642, "process": {"exists": false, "pid": 22642}, "restart_count": 177, "restart_count_per_hour": 28, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 178}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.82, "sample_count": 2186, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 87946, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:40:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-01T20:40:25+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-01T20:40:25+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-01T20:40:25.089887+09:00", "mtime_epoch": 1782906025.0898867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 147249}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:40:25+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 903.332, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 221, "pushed_frames": 221, "read_frames": 220, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.391, "pushed": 7.391, "read": 7.357}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.417, "count": 221, "max_ms": 6.244, "p95_ms": 5.093}, "gstreamer_push": {"avg_ms": 1.247, "count": 221, "max_ms": 4.429, "p95_ms": 2.104}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.382, "count": 221, "max_ms": 80.339, "p95_ms": 65.96}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.555, "count": 221, "max_ms": 92.312, "p95_ms": 87.799}, "yolo11_640_fire_11class": {"avg_ms": 56.394, "count": 221, "max_ms": 78.676, "p95_ms": 74.724}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.92, "count": 221, "max_ms": 91.067, "p95_ms": 86.816}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.151, "count": 221, "max_ms": 191.363, "p95_ms": 150.743}, "read_interval": {"avg_ms": 135.808, "count": 220, "max_ms": 204.323, "p95_ms": 162.087}, "triton_wait": {"avg_ms": 66.539, "count": 221, "max_ms": 92.323, "p95_ms": 88.324}, "visualize": {"avg_ms": 5.416, "count": 221, "max_ms": 9.347, "p95_ms": 6.497}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 732}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4074.887, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:36:56+09:00", "last_error_summary": "[h264 @ 0x55b8807a2540] error while decoding MB 41 41, bytestream -5", "last_exit_at": "2026-07-01T20:40:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 205.833, "last_started_at": "2026-07-01T20:36:59+09:00", "pid": 22642, "process": {"exists": false, "pid": 22642}, "restart_count": 177, "restart_count_per_hour": 28, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 178}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.82, "sample_count": 2186, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 87946, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:40:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T20:40:25+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-01T20:40:25+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-01T20:40:25.089887+09:00", "mtime_epoch": 1782906025.0898867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 147249}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:40:25+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 903.332, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 221, "pushed_frames": 221, "read_frames": 220, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.39, "pushed": 7.39, "read": 7.356}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.417, "count": 221, "max_ms": 6.244, "p95_ms": 5.093}, "gstreamer_push": {"avg_ms": 1.247, "count": 221, "max_ms": 4.429, "p95_ms": 2.104}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.382, "count": 221, "max_ms": 80.339, "p95_ms": 65.96}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.555, "count": 221, "max_ms": 92.312, "p95_ms": 87.799}, "yolo11_640_fire_11class": {"avg_ms": 56.394, "count": 221, "max_ms": 78.676, "p95_ms": 74.724}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.92, "count": 221, "max_ms": 91.067, "p95_ms": 86.816}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.151, "count": 221, "max_ms": 191.363, "p95_ms": 150.743}, "read_interval": {"avg_ms": 135.808, "count": 220, "max_ms": 204.323, "p95_ms": 162.087}, "triton_wait": {"avg_ms": 66.539, "count": 221, "max_ms": 92.323, "p95_ms": 88.324}, "visualize": {"avg_ms": 5.416, "count": 221, "max_ms": 9.347, "p95_ms": 6.497}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 732}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4187.781, "limit_mb": 8796093022207.996}, "detail": {"pid": 22749, "restart_count": 178, "start_count": 179}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 153, "h264_decode_error_count": 159, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:36:56+09:00", "last_error_summary": "[h264 @ 0x55b8807a2540] error while decoding MB 41 41, bytestream -5", "last_exit_at": "2026-07-01T20:40:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:40:28+09:00", "pid": 22749, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.852, "name": "ffmpeg", "pid": 22749, "rss_mb": 141.852, "state": "S (sleeping)", "threads": 106}, "restart_count": 178, "restart_count_per_hour": 29, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 179}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.226, "sample_count": 2149, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 87946, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:40:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.908, "last_frame_pushed_at": "2026-07-01T20:40:25+09:00", "last_frame_read_age_sec": 5.043, "last_frame_read_at": "2026-07-01T20:40:25+09:00", "mjpeg": {"age_sec": 4.891, "exists": true, "mtime": "2026-07-01T20:40:25.233891+09:00", "mtime_epoch": 1782906025.233891, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 147190}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:40:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.086, "name": "celery", "pid": 237, "rss_mb": 903.332, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 185, "pushed_frames": 185, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.195, "pushed": 6.195, "read": 6.162}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.409, "count": 185, "max_ms": 6.244, "p95_ms": 5.093}, "gstreamer_push": {"avg_ms": 1.255, "count": 185, "max_ms": 4.429, "p95_ms": 2.255}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.98, "count": 185, "max_ms": 80.339, "p95_ms": 65.96}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.388, "count": 185, "max_ms": 92.312, "p95_ms": 87.069}, "yolo11_640_fire_11class": {"avg_ms": 56.309, "count": 185, "max_ms": 76.505, "p95_ms": 74.724}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.022, "count": 185, "max_ms": 91.067, "p95_ms": 86.592}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.975, "count": 185, "max_ms": 191.363, "p95_ms": 150.507}, "read_interval": {"avg_ms": 135.644, "count": 184, "max_ms": 204.323, "p95_ms": 162.081}, "triton_wait": {"avg_ms": 66.504, "count": 185, "max_ms": 92.323, "p95_ms": 87.793}, "visualize": {"avg_ms": 5.413, "count": 185, "max_ms": 9.347, "p95_ms": 6.5}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.862, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 748}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4079.84, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55633c7bc4c0] error while decoding MB 38 16, bytestream -28", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 154, "h264_decode_error_count": 160, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:44:28+09:00", "last_error_summary": "[h264 @ 0x55633c7bc4c0] error while decoding MB 38 16, bytestream -28", "last_exit_at": "2026-07-01T20:40:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.208, "last_started_at": "2026-07-01T20:40:28+09:00", "pid": 22749, "process": {"exists": false, "pid": 22749}, "restart_count": 178, "restart_count_per_hour": 27, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 179}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.154, "sample_count": 2299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 89828, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:44:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-01T20:44:28+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T20:44:28+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-01T20:44:28.280664+09:00", "mtime_epoch": 1782906268.2806637, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120113}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:44:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.199, "name": "celery", "pid": 237, "rss_mb": 903.922, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 246, "pushed_frames": 246, "read_frames": 247, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.2, "pushed": 8.2, "read": 8.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.219, "count": 246, "max_ms": 5.687, "p95_ms": 4.549}, "gstreamer_push": {"avg_ms": 1.369, "count": 246, "max_ms": 3.989, "p95_ms": 2.606}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.761, "count": 246, "max_ms": 76.223, "p95_ms": 67.61}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.767, "count": 246, "max_ms": 88.523, "p95_ms": 85.031}, "yolo11_640_fire_11class": {"avg_ms": 61.21, "count": 246, "max_ms": 81.769, "p95_ms": 75.487}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.841, "count": 246, "max_ms": 90.13, "p95_ms": 87.193}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.674, "count": 246, "max_ms": 155.428, "p95_ms": 138.548}, "read_interval": {"avg_ms": 121.716, "count": 247, "max_ms": 170.273, "p95_ms": 152.477}, "triton_wait": {"avg_ms": 69.715, "count": 246, "max_ms": 90.142, "p95_ms": 87.567}, "visualize": {"avg_ms": 4.721, "count": 246, "max_ms": 9.735, "p95_ms": 5.579}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 748}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4078.879, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:44:28+09:00", "last_error_summary": "[h264 @ 0x55633c7bc4c0] error while decoding MB 38 16, bytestream -28", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 154, "h264_decode_error_count": 160, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:44:28+09:00", "last_error_summary": "[h264 @ 0x55633c7bc4c0] error while decoding MB 38 16, bytestream -28", "last_exit_at": "2026-07-01T20:44:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.297, "last_started_at": "2026-07-01T20:40:28+09:00", "pid": 22749, "process": {"exists": false, "pid": 22749}, "restart_count": 178, "restart_count_per_hour": 27, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 179}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.152, "sample_count": 2299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 89829, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:44:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T20:44:28+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-01T20:44:28+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-01T20:44:28.280664+09:00", "mtime_epoch": 1782906268.2806637, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120113}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:44:28+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.199, "name": "celery", "pid": 237, "rss_mb": 903.922, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 247, "pushed_frames": 247, "read_frames": 246, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.246, "pushed": 8.246, "read": 8.213}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.219, "count": 246, "max_ms": 5.687, "p95_ms": 4.549}, "gstreamer_push": {"avg_ms": 1.368, "count": 247, "max_ms": 3.989, "p95_ms": 2.606}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.866, "count": 246, "max_ms": 76.223, "p95_ms": 67.61}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.841, "count": 246, "max_ms": 88.523, "p95_ms": 85.031}, "yolo11_640_fire_11class": {"avg_ms": 61.28, "count": 246, "max_ms": 81.769, "p95_ms": 75.487}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.929, "count": 246, "max_ms": 90.13, "p95_ms": 87.193}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.788, "count": 247, "max_ms": 155.428, "p95_ms": 138.548}, "read_interval": {"avg_ms": 121.724, "count": 246, "max_ms": 170.273, "p95_ms": 152.477}, "triton_wait": {"avg_ms": 69.803, "count": 246, "max_ms": 90.142, "p95_ms": 87.567}, "visualize": {"avg_ms": 4.721, "count": 246, "max_ms": 9.735, "p95_ms": 5.579}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 748}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4078.992, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 154, "h264_decode_error_count": 160, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:44:28+09:00", "last_error_summary": "[h264 @ 0x55633c7bc4c0] error while decoding MB 38 16, bytestream -28", "last_exit_at": "2026-07-01T20:44:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.303, "last_started_at": "2026-07-01T20:40:28+09:00", "pid": 22749, "process": {"exists": false, "pid": 22749}, "restart_count": 178, "restart_count_per_hour": 27, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 179}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.152, "sample_count": 2299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 89829, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:44:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T20:44:28+09:00", "last_frame_read_age_sec": 0.149, "last_frame_read_at": "2026-07-01T20:44:28+09:00", "mjpeg": {"age_sec": 0.146, "exists": true, "mtime": "2026-07-01T20:44:28.280664+09:00", "mtime_epoch": 1782906268.2806637, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120113}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:44:28+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.199, "name": "celery", "pid": 237, "rss_mb": 903.922, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 247, "pushed_frames": 247, "read_frames": 246, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.245, "pushed": 8.245, "read": 8.211}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.219, "count": 246, "max_ms": 5.687, "p95_ms": 4.549}, "gstreamer_push": {"avg_ms": 1.368, "count": 247, "max_ms": 3.989, "p95_ms": 2.606}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.866, "count": 246, "max_ms": 76.223, "p95_ms": 67.61}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.841, "count": 246, "max_ms": 88.523, "p95_ms": 85.031}, "yolo11_640_fire_11class": {"avg_ms": 61.28, "count": 246, "max_ms": 81.769, "p95_ms": 75.487}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.929, "count": 246, "max_ms": 90.13, "p95_ms": 87.193}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.788, "count": 247, "max_ms": 155.428, "p95_ms": 138.548}, "read_interval": {"avg_ms": 121.724, "count": 246, "max_ms": 170.273, "p95_ms": 152.477}, "triton_wait": {"avg_ms": 69.803, "count": 246, "max_ms": 90.142, "p95_ms": 87.567}, "visualize": {"avg_ms": 4.721, "count": 246, "max_ms": 9.735, "p95_ms": 5.579}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 748}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4184.32, "limit_mb": 8796093022207.996}, "detail": {"pid": 22856, "restart_count": 179, "start_count": 180}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 154, "h264_decode_error_count": 160, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:44:28+09:00", "last_error_summary": "[h264 @ 0x55633c7bc4c0] error while decoding MB 38 16, bytestream -28", "last_exit_at": "2026-07-01T20:44:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:44:31+09:00", "pid": 22856, "process": {"exists": true, "fd_count": 5, "hwm_mb": 134.094, "name": "ffmpeg", "pid": 22856, "rss_mb": 134.094, "state": "S (sleeping)", "threads": 106}, "restart_count": 179, "restart_count_per_hour": 28, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 180}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.032, "sample_count": 2266, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 89829, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:44:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.633, "last_frame_pushed_at": "2026-07-01T20:44:28+09:00", "last_frame_read_age_sec": 4.771, "last_frame_read_at": "2026-07-01T20:44:28+09:00", "mjpeg": {"age_sec": 4.616, "exists": true, "mtime": "2026-07-01T20:44:28.432667+09:00", "mtime_epoch": 1782906268.4326675, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119343}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:44:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.199, "name": "celery", "pid": 237, "rss_mb": 903.922, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.953, "pushed": 6.953, "read": 6.919}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.188, "count": 208, "max_ms": 5.685, "p95_ms": 4.252}, "gstreamer_push": {"avg_ms": 1.35, "count": 208, "max_ms": 3.989, "p95_ms": 2.459}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.828, "count": 208, "max_ms": 76.223, "p95_ms": 67.61}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.33, "count": 208, "max_ms": 88.523, "p95_ms": 85.279}, "yolo11_640_fire_11class": {"avg_ms": 61.784, "count": 208, "max_ms": 81.769, "p95_ms": 75.855}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.57, "count": 208, "max_ms": 90.13, "p95_ms": 87.36}}, "truncated_count": 0}, "processing_total": {"avg_ms": 108.16, "count": 208, "max_ms": 146.799, "p95_ms": 138.458}, "read_interval": {"avg_ms": 122.008, "count": 207, "max_ms": 160.947, "p95_ms": 152.284}, "triton_wait": {"avg_ms": 70.404, "count": 208, "max_ms": 90.142, "p95_ms": 87.72}, "visualize": {"avg_ms": 4.661, "count": 208, "max_ms": 6.956, "p95_ms": 5.466}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 763}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4078.305, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d8802ba540] cabac decode of qscale diff failed at 19 43 [h264 @ 0x55d8802ba540] error while decoding MB 19 43, bytestream 3571 [h264 @ 0x55d8805270c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8805270c0] error while decoding MB 0 2, bytestream 6569 [h264 @ 0x55d880458240] error while decoding MB 2 38, bytestream -12 [h264 @ 0x55d87f870840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d87f870840] error while decoding MB 0 15, bytestream 41", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 155, "h264_decode_error_count": 167, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:48:29+09:00", "last_error_summary": "[h264 @ 0x55d8802ba540] cabac decode of qscale diff failed at 19 43 [h264 @ 0x55d8802ba540] error while decoding MB 19 43, bytestream 3571 [h264 @ 0x55d8805270c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8805270c0] error while decoding MB 0 2, bytestream 6569 [h264 @ 0x55d880458240] error while decoding MB 2 38, bytestream -12 [h264 @ 0x55d87f870840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d87f870840] error while decoding MB 0 15, bytestream 41", "last_exit_at": "2026-07-01T20:44:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.28, "last_started_at": "2026-07-01T20:44:31+09:00", "pid": 22856, "process": {"exists": false, "pid": 22856}, "restart_count": 179, "restart_count_per_hour": 26, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 180}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.196, "sample_count": 2237, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 91582, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:48:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.089, "last_frame_pushed_at": "2026-07-01T20:48:29+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-01T20:48:29+09:00", "mjpeg": {"age_sec": 0.065, "exists": true, "mtime": "2026-07-01T20:48:29.262711+09:00", "mtime_epoch": 1782906509.2627108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115137}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:48:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.199, "name": "celery", "pid": 237, "rss_mb": 901.953, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.101, "pushed": 4.101, "read": 4.101}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.758, "count": 123, "max_ms": 4.552, "p95_ms": 4.004}, "gstreamer_push": {"avg_ms": 1.434, "count": 123, "max_ms": 4.372, "p95_ms": 3.008}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.896, "count": 123, "max_ms": 69.487, "p95_ms": 61.797}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.795, "count": 123, "max_ms": 82.638, "p95_ms": 75.66}, "yolo11_640_fire_11class": {"avg_ms": 55.526, "count": 123, "max_ms": 76.152, "p95_ms": 72.028}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.856, "count": 123, "max_ms": 86.478, "p95_ms": 83.892}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.874, "count": 123, "max_ms": 153.443, "p95_ms": 134.56}, "read_interval": {"avg_ms": 243.781, "count": 123, "max_ms": 17093.109, "p95_ms": 148.707}, "triton_wait": {"avg_ms": 60.627, "count": 123, "max_ms": 86.489, "p95_ms": 83.903}, "visualize": {"avg_ms": 4.427, "count": 123, "max_ms": 6.047, "p95_ms": 5.276}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 763}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4078.105, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:48:29+09:00", "last_error_summary": "[h264 @ 0x55d8802ba540] cabac decode of qscale diff failed at 19 43 [h264 @ 0x55d8802ba540] error while decoding MB 19 43, bytestream 3571 [h264 @ 0x55d8805270c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8805270c0] error while decoding MB 0 2, bytestream 6569 [h264 @ 0x55d880458240] error while decoding MB 2 38, bytestream -12 [h264 @ 0x55d87f870840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d87f870840] error while decoding MB 0 15, bytestream 41", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 155, "h264_decode_error_count": 167, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:48:29+09:00", "last_error_summary": "[h264 @ 0x55d8802ba540] cabac decode of qscale diff failed at 19 43 [h264 @ 0x55d8802ba540] error while decoding MB 19 43, bytestream 3571 [h264 @ 0x55d8805270c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8805270c0] error while decoding MB 0 2, bytestream 6569 [h264 @ 0x55d880458240] error while decoding MB 2 38, bytestream -12 [h264 @ 0x55d87f870840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d87f870840] error while decoding MB 0 15, bytestream 41", "last_exit_at": "2026-07-01T20:48:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.308, "last_started_at": "2026-07-01T20:44:31+09:00", "pid": 22856, "process": {"exists": false, "pid": 22856}, "restart_count": 179, "restart_count_per_hour": 26, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 180}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.196, "sample_count": 2238, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 91583, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:48:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:48:29+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-01T20:48:29+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-01T20:48:29.262711+09:00", "mtime_epoch": 1782906509.2627108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115137}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:48:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.199, "name": "celery", "pid": 237, "rss_mb": 901.953, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.113, "pushed": 4.113, "read": 4.08}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.777, "count": 123, "max_ms": 4.552, "p95_ms": 4.004}, "gstreamer_push": {"avg_ms": 1.434, "count": 123, "max_ms": 4.372, "p95_ms": 3.008}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.843, "count": 123, "max_ms": 69.487, "p95_ms": 61.797}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.016, "count": 123, "max_ms": 82.638, "p95_ms": 75.728}, "yolo11_640_fire_11class": {"avg_ms": 55.599, "count": 123, "max_ms": 76.152, "p95_ms": 72.028}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.072, "count": 123, "max_ms": 86.478, "p95_ms": 83.892}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.114, "count": 123, "max_ms": 153.443, "p95_ms": 134.56}, "read_interval": {"avg_ms": 245.113, "count": 122, "max_ms": 17093.109, "p95_ms": 148.707}, "triton_wait": {"avg_ms": 60.842, "count": 123, "max_ms": 86.489, "p95_ms": 83.903}, "visualize": {"avg_ms": 4.431, "count": 123, "max_ms": 6.047, "p95_ms": 5.276}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 763}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4078.148, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 155, "h264_decode_error_count": 167, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:48:29+09:00", "last_error_summary": "[h264 @ 0x55d8802ba540] cabac decode of qscale diff failed at 19 43 [h264 @ 0x55d8802ba540] error while decoding MB 19 43, bytestream 3571 [h264 @ 0x55d8805270c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8805270c0] error while decoding MB 0 2, bytestream 6569 [h264 @ 0x55d880458240] error while decoding MB 2 38, bytestream -12 [h264 @ 0x55d87f870840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d87f870840] error while decoding MB 0 15, bytestream 41", "last_exit_at": "2026-07-01T20:48:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.316, "last_started_at": "2026-07-01T20:44:31+09:00", "pid": 22856, "process": {"exists": false, "pid": 22856}, "restart_count": 179, "restart_count_per_hour": 26, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 180}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.196, "sample_count": 2238, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 91583, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:48:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T20:48:29+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T20:48:29+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-01T20:48:29.262711+09:00", "mtime_epoch": 1782906509.2627108, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115137}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:48:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.199, "name": "celery", "pid": 237, "rss_mb": 901.953, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.112, "pushed": 4.112, "read": 4.079}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.777, "count": 123, "max_ms": 4.552, "p95_ms": 4.004}, "gstreamer_push": {"avg_ms": 1.434, "count": 123, "max_ms": 4.372, "p95_ms": 3.008}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.843, "count": 123, "max_ms": 69.487, "p95_ms": 61.797}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.016, "count": 123, "max_ms": 82.638, "p95_ms": 75.728}, "yolo11_640_fire_11class": {"avg_ms": 55.599, "count": 123, "max_ms": 76.152, "p95_ms": 72.028}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.072, "count": 123, "max_ms": 86.478, "p95_ms": 83.892}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.114, "count": 123, "max_ms": 153.443, "p95_ms": 134.56}, "read_interval": {"avg_ms": 245.113, "count": 122, "max_ms": 17093.109, "p95_ms": 148.707}, "triton_wait": {"avg_ms": 60.842, "count": 123, "max_ms": 86.489, "p95_ms": 83.903}, "visualize": {"avg_ms": 4.431, "count": 123, "max_ms": 6.047, "p95_ms": 5.276}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 763}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4182.898, "limit_mb": 8796093022207.996}, "detail": {"pid": 22963, "restart_count": 180, "start_count": 181}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 155, "h264_decode_error_count": 167, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:48:29+09:00", "last_error_summary": "[h264 @ 0x55d8802ba540] cabac decode of qscale diff failed at 19 43 [h264 @ 0x55d8802ba540] error while decoding MB 19 43, bytestream 3571 [h264 @ 0x55d8805270c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d8805270c0] error while decoding MB 0 2, bytestream 6569 [h264 @ 0x55d880458240] error while decoding MB 2 38, bytestream -12 [h264 @ 0x55d87f870840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d87f870840] error while decoding MB 0 15, bytestream 41", "last_exit_at": "2026-07-01T20:48:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:48:32+09:00", "pid": 22963, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.57, "name": "ffmpeg", "pid": 22963, "rss_mb": 138.57, "state": "S (sleeping)", "threads": 106}, "restart_count": 180, "restart_count_per_hour": 27, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 181}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.196, "sample_count": 2194, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 91583, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:48:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.808, "last_frame_pushed_at": "2026-07-01T20:48:29+09:00", "last_frame_read_age_sec": 4.906, "last_frame_read_at": "2026-07-01T20:48:29+09:00", "mjpeg": {"age_sec": 4.791, "exists": true, "mtime": "2026-07-01T20:48:29.366713+09:00", "mtime_epoch": 1782906509.3667133, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114384}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:48:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.199, "name": "celery", "pid": 237, "rss_mb": 901.953, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 16, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.872, "pushed": 2.872, "read": 2.839}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.274, "count": 86, "max_ms": 4.11, "p95_ms": 3.957}, "gstreamer_push": {"avg_ms": 1.42, "count": 86, "max_ms": 3.866, "p95_ms": 2.332}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.361, "count": 86, "max_ms": 69.487, "p95_ms": 60.701}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.38, "count": 86, "max_ms": 82.638, "p95_ms": 75.66}, "yolo11_640_fire_11class": {"avg_ms": 54.904, "count": 86, "max_ms": 74.44, "p95_ms": 70.776}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.517, "count": 86, "max_ms": 84.933, "p95_ms": 82.782}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.943, "count": 86, "max_ms": 135.386, "p95_ms": 132.104}, "read_interval": {"avg_ms": 295.734, "count": 85, "max_ms": 17093.109, "p95_ms": 144.927}, "triton_wait": {"avg_ms": 59.589, "count": 86, "max_ms": 84.946, "p95_ms": 82.793}, "visualize": {"avg_ms": 4.132, "count": 86, "max_ms": 5.654, "p95_ms": 5.187}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4081.129, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5654b1194d80] reference picture missing during reorder [h264 @ 0x5654b1194d80] Missing reference picture, default is 65580 [h264 @ 0x5654b19e9500] mmco: unref short failure [h264 @ 0x5654b19e9500] illegal short term buffer state detected [h264 @ 0x5654b1158900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5654b1158900] error while decoding MB 0 39, bytestream 43 [h264 @ 0x5654b1052740] left block unavailable for requested intra mode [h264 @ 0x5654b1052740] error whil", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 156, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:57:10+09:00", "last_error_summary": "[h264 @ 0x5654b1194d80] reference picture missing during reorder [h264 @ 0x5654b1194d80] Missing reference picture, default is 65580 [h264 @ 0x5654b19e9500] mmco: unref short failure [h264 @ 0x5654b19e9500] illegal short term buffer state detected [h264 @ 0x5654b1158900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5654b1158900] error while decoding MB 0 39, bytestream 43 [h264 @ 0x5654b1052740] left block unavailable for requested intra mode [h264 @ 0x5654b1052740] error whil", "last_exit_at": "2026-07-01T20:48:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 517.886, "last_started_at": "2026-07-01T20:48:32+09:00", "pid": 22963, "process": {"exists": false, "pid": 22963}, "restart_count": 180, "restart_count_per_hour": 22, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 181}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.496, "sample_count": 2577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 96267, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:57:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-01T20:57:09+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-01T20:57:09+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T20:57:09.994565+09:00", "mtime_epoch": 1782907029.994565, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114243}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:57:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.438, "name": "celery", "pid": 237, "rss_mb": 906.602, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.31, "pushed": 3.31, "read": 3.31}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.598, "count": 99, "max_ms": 2.128, "p95_ms": 1.976}, "gstreamer_push": {"avg_ms": 1.528, "count": 99, "max_ms": 6.311, "p95_ms": 3.494}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.263, "count": 99, "max_ms": 80.466, "p95_ms": 77.464}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.55, "count": 99, "max_ms": 82.337, "p95_ms": 79.125}, "yolo11_640_fire_11class": {"avg_ms": 71.203, "count": 99, "max_ms": 79.81, "p95_ms": 77.032}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.396, "count": 99, "max_ms": 92.722, "p95_ms": 86.829}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.111, "count": 99, "max_ms": 103.339, "p95_ms": 97.267}, "read_interval": {"avg_ms": 302.577, "count": 99, "max_ms": 10126.786, "p95_ms": 116.688}, "triton_wait": {"avg_ms": 79.637, "count": 99, "max_ms": 92.735, "p95_ms": 86.839}, "visualize": {"avg_ms": 3.716, "count": 99, "max_ms": 4.805, "p95_ms": 4.057}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4078.301, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:57:10+09:00", "last_error_summary": "[h264 @ 0x5654b1194d80] reference picture missing during reorder [h264 @ 0x5654b1194d80] Missing reference picture, default is 65580 [h264 @ 0x5654b19e9500] mmco: unref short failure [h264 @ 0x5654b19e9500] illegal short term buffer state detected [h264 @ 0x5654b1158900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5654b1158900] error while decoding MB 0 39, bytestream 43 [h264 @ 0x5654b1052740] left block unavailable for requested intra mode [h264 @ 0x5654b1052740] error whil", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 156, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:57:10+09:00", "last_error_summary": "[h264 @ 0x5654b1194d80] reference picture missing during reorder [h264 @ 0x5654b1194d80] Missing reference picture, default is 65580 [h264 @ 0x5654b19e9500] mmco: unref short failure [h264 @ 0x5654b19e9500] illegal short term buffer state detected [h264 @ 0x5654b1158900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5654b1158900] error while decoding MB 0 39, bytestream 43 [h264 @ 0x5654b1052740] left block unavailable for requested intra mode [h264 @ 0x5654b1052740] error whil", "last_exit_at": "2026-07-01T20:57:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 517.911, "last_started_at": "2026-07-01T20:48:32+09:00", "pid": 22963, "process": {"exists": false, "pid": 22963}, "restart_count": 180, "restart_count_per_hour": 22, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 181}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.496, "sample_count": 2578, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 96268, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:57:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:57:10+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-01T20:57:09+09:00", "mjpeg": {"age_sec": 0.073, "exists": true, "mtime": "2026-07-01T20:57:09.994565+09:00", "mtime_epoch": 1782907029.994565, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114243}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:57:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.438, "name": "celery", "pid": 237, "rss_mb": 903.766, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.34, "pushed": 3.34, "read": 3.307}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.597, "count": 100, "max_ms": 2.128, "p95_ms": 1.938}, "gstreamer_push": {"avg_ms": 1.575, "count": 100, "max_ms": 6.311, "p95_ms": 3.494}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.103, "count": 100, "max_ms": 80.466, "p95_ms": 76.802}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.307, "count": 100, "max_ms": 82.337, "p95_ms": 79.118}, "yolo11_640_fire_11class": {"avg_ms": 70.959, "count": 100, "max_ms": 79.81, "p95_ms": 77.029}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.075, "count": 100, "max_ms": 92.722, "p95_ms": 86.054}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.844, "count": 100, "max_ms": 103.339, "p95_ms": 96.56}, "read_interval": {"avg_ms": 302.577, "count": 99, "max_ms": 10126.786, "p95_ms": 116.688}, "triton_wait": {"avg_ms": 79.323, "count": 100, "max_ms": 92.735, "p95_ms": 86.067}, "visualize": {"avg_ms": 3.743, "count": 100, "max_ms": 6.375, "p95_ms": 4.057}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4078.418, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 156, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:57:10+09:00", "last_error_summary": "[h264 @ 0x5654b1194d80] reference picture missing during reorder [h264 @ 0x5654b1194d80] Missing reference picture, default is 65580 [h264 @ 0x5654b19e9500] mmco: unref short failure [h264 @ 0x5654b19e9500] illegal short term buffer state detected [h264 @ 0x5654b1158900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5654b1158900] error while decoding MB 0 39, bytestream 43 [h264 @ 0x5654b1052740] left block unavailable for requested intra mode [h264 @ 0x5654b1052740] error whil", "last_exit_at": "2026-07-01T20:57:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 517.919, "last_started_at": "2026-07-01T20:48:32+09:00", "pid": 22963, "process": {"exists": false, "pid": 22963}, "restart_count": 180, "restart_count_per_hour": 22, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 181}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.496, "sample_count": 2578, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 96268, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:57:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T20:57:10+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-01T20:57:09+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-01T20:57:09.994565+09:00", "mtime_epoch": 1782907029.994565, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114243}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:57:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 909.438, "name": "celery", "pid": 237, "rss_mb": 903.766, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.34, "pushed": 3.34, "read": 3.306}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.597, "count": 100, "max_ms": 2.128, "p95_ms": 1.938}, "gstreamer_push": {"avg_ms": 1.575, "count": 100, "max_ms": 6.311, "p95_ms": 3.494}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.103, "count": 100, "max_ms": 80.466, "p95_ms": 76.802}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.307, "count": 100, "max_ms": 82.337, "p95_ms": 79.118}, "yolo11_640_fire_11class": {"avg_ms": 70.959, "count": 100, "max_ms": 79.81, "p95_ms": 77.029}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.075, "count": 100, "max_ms": 92.722, "p95_ms": 86.054}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.844, "count": 100, "max_ms": 103.339, "p95_ms": 96.56}, "read_interval": {"avg_ms": 302.577, "count": 99, "max_ms": 10126.786, "p95_ms": 116.688}, "triton_wait": {"avg_ms": 79.323, "count": 100, "max_ms": 92.735, "p95_ms": 86.067}, "visualize": {"avg_ms": 3.743, "count": 100, "max_ms": 6.375, "p95_ms": 4.057}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1795}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3791.746, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55591f199a40] error while decoding MB 93 3, bytestream -5 [in#0/rtsp @ 0x55591f086e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55591f086e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:57:13+09:00", "last_error_summary": "[h264 @ 0x55591f199a40] error while decoding MB 93 3, bytestream -5 [in#0/rtsp @ 0x55591f086e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55591f086e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T16:35:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 15700.458, "last_started_at": "2026-07-01T16:35:32+09:00", "pid": 9162, "process": {"exists": false, "pid": 9162}, "restart_count": 7, "restart_count_per_hour": 0, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.051, "sample_count": 2584, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 215420, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:57:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T20:57:13+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T20:57:13+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-01T20:57:13.038631+09:00", "mtime_epoch": 1782907033.0386314, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99145}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T20:57:13+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 967.16, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 104, "read_frames": 104, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.5, "pushed": 3.467, "read": 3.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.652, "count": 104, "max_ms": 4.809, "p95_ms": 4.261}, "gstreamer_push": {"avg_ms": 1.704, "count": 104, "max_ms": 6.034, "p95_ms": 3.796}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.478, "count": 104, "max_ms": 77.696, "p95_ms": 75.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.011, "count": 104, "max_ms": 78.187, "p95_ms": 75.556}, "yolo11_640_fire_11class": {"avg_ms": 60.648, "count": 104, "max_ms": 77.043, "p95_ms": 74.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.841, "count": 104, "max_ms": 89.862, "p95_ms": 84.823}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.797, "count": 105, "max_ms": 102.334, "p95_ms": 97.47}, "read_interval": {"avg_ms": 288.627, "count": 104, "max_ms": 10769.353, "p95_ms": 115.959}, "triton_wait": {"avg_ms": 66.996, "count": 104, "max_ms": 89.874, "p95_ms": 84.835}, "visualize": {"avg_ms": 3.704, "count": 104, "max_ms": 4.605, "p95_ms": 4.061}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1795}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3791.605, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:57:13+09:00", "last_error_summary": "[h264 @ 0x55591f199a40] error while decoding MB 93 3, bytestream -5 [in#0/rtsp @ 0x55591f086e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55591f086e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:57:13+09:00", "last_error_summary": "[h264 @ 0x55591f199a40] error while decoding MB 93 3, bytestream -5 [in#0/rtsp @ 0x55591f086e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55591f086e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T20:57:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 15700.475, "last_started_at": "2026-07-01T16:35:32+09:00", "pid": 9162, "process": {"exists": false, "pid": 9162}, "restart_count": 7, "restart_count_per_hour": 0, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.051, "sample_count": 2584, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 215420, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:57:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T20:57:13+09:00", "last_frame_read_age_sec": 0.081, "last_frame_read_at": "2026-07-01T20:57:13+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-01T20:57:13.038631+09:00", "mtime_epoch": 1782907033.0386314, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99145}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T20:57:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 967.16, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 104, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.467, "pushed": 3.467, "read": 3.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.652, "count": 104, "max_ms": 4.809, "p95_ms": 4.261}, "gstreamer_push": {"avg_ms": 1.742, "count": 104, "max_ms": 6.034, "p95_ms": 3.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.478, "count": 104, "max_ms": 77.696, "p95_ms": 75.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.011, "count": 104, "max_ms": 78.187, "p95_ms": 75.556}, "yolo11_640_fire_11class": {"avg_ms": 60.648, "count": 104, "max_ms": 77.043, "p95_ms": 74.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.841, "count": 104, "max_ms": 89.862, "p95_ms": 84.823}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.748, "count": 104, "max_ms": 102.334, "p95_ms": 97.47}, "read_interval": {"avg_ms": 288.627, "count": 104, "max_ms": 10769.353, "p95_ms": 115.959}, "triton_wait": {"avg_ms": 66.996, "count": 104, "max_ms": 89.874, "p95_ms": 84.835}, "visualize": {"avg_ms": 3.704, "count": 104, "max_ms": 4.605, "p95_ms": 4.061}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1795}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3791.469, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:57:13+09:00", "last_error_summary": "[h264 @ 0x55591f199a40] error while decoding MB 93 3, bytestream -5 [in#0/rtsp @ 0x55591f086e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55591f086e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T20:57:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 15700.483, "last_started_at": "2026-07-01T16:35:32+09:00", "pid": 9162, "process": {"exists": false, "pid": 9162}, "restart_count": 7, "restart_count_per_hour": 0, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.051, "sample_count": 2583, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 215420, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:57:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.02, "last_frame_pushed_at": "2026-07-01T20:57:13+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-01T20:57:13+09:00", "mjpeg": {"age_sec": 0.009, "exists": true, "mtime": "2026-07-01T20:57:13.114633+09:00", "mtime_epoch": 1782907033.114633, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97020}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T20:57:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 967.16, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.477, "pushed": 3.477, "read": 3.443}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.652, "count": 104, "max_ms": 4.809, "p95_ms": 4.261}, "gstreamer_push": {"avg_ms": 1.742, "count": 104, "max_ms": 6.034, "p95_ms": 3.898}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.478, "count": 104, "max_ms": 77.696, "p95_ms": 75.063}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.011, "count": 104, "max_ms": 78.187, "p95_ms": 75.556}, "yolo11_640_fire_11class": {"avg_ms": 60.648, "count": 104, "max_ms": 77.043, "p95_ms": 74.312}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.841, "count": 104, "max_ms": 89.862, "p95_ms": 84.823}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.748, "count": 104, "max_ms": 102.334, "p95_ms": 97.47}, "read_interval": {"avg_ms": 290.446, "count": 103, "max_ms": 10769.353, "p95_ms": 115.959}, "triton_wait": {"avg_ms": 66.996, "count": 104, "max_ms": 89.874, "p95_ms": 84.835}, "visualize": {"avg_ms": 3.704, "count": 104, "max_ms": 4.605, "p95_ms": 4.061}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3887.0, "limit_mb": 8796093022207.996}, "detail": {"pid": 23070, "restart_count": 181, "start_count": 182}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 156, "h264_decode_error_count": 170, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:57:10+09:00", "last_error_summary": "[h264 @ 0x5654b1194d80] reference picture missing during reorder [h264 @ 0x5654b1194d80] Missing reference picture, default is 65580 [h264 @ 0x5654b19e9500] mmco: unref short failure [h264 @ 0x5654b19e9500] illegal short term buffer state detected [h264 @ 0x5654b1158900] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5654b1158900] error while decoding MB 0 39, bytestream 43 [h264 @ 0x5654b1052740] left block unavailable for requested intra mode [h264 @ 0x5654b1052740] error whil", "last_exit_at": "2026-07-01T20:57:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:57:12+09:00", "pid": 23070, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.02, "name": "ffmpeg", "pid": 23070, "rss_mb": 139.02, "state": "S (sleeping)", "threads": 106}, "restart_count": 181, "restart_count_per_hour": 23, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 182}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.496, "sample_count": 2526, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 96268, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:57:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.862, "last_frame_pushed_at": "2026-07-01T20:57:10+09:00", "last_frame_read_age_sec": 4.934, "last_frame_read_at": "2026-07-01T20:57:09+09:00", "mjpeg": {"age_sec": 4.844, "exists": true, "mtime": "2026-07-01T20:57:10.078567+09:00", "mtime_epoch": 1782907030.0785668, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114773}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T20:57:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 909.438, "name": "celery", "pid": 237, "rss_mb": 903.766, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.87, "pushed": 1.87, "read": 1.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.593, "count": 56, "max_ms": 2.128, "p95_ms": 1.938}, "gstreamer_push": {"avg_ms": 1.459, "count": 56, "max_ms": 6.226, "p95_ms": 1.472}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.793, "count": 56, "max_ms": 80.466, "p95_ms": 77.663}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.402, "count": 56, "max_ms": 82.337, "p95_ms": 79.542}, "yolo11_640_fire_11class": {"avg_ms": 68.773, "count": 56, "max_ms": 79.81, "p95_ms": 77.032}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.101, "count": 56, "max_ms": 92.722, "p95_ms": 87.392}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.988, "count": 56, "max_ms": 103.339, "p95_ms": 97.839}, "read_interval": {"avg_ms": 456.52, "count": 55, "max_ms": 10126.786, "p95_ms": 119.044}, "triton_wait": {"avg_ms": 76.521, "count": 56, "max_ms": 92.735, "p95_ms": 87.404}, "visualize": {"avg_ms": 3.726, "count": 56, "max_ms": 6.375, "p95_ms": 3.93}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1795}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4128.633, "limit_mb": 8796093022207.996}, "detail": {"pid": 23177, "restart_count": 8, "start_count": 9}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:57:13+09:00", "last_error_summary": "[h264 @ 0x55591f199a40] error while decoding MB 93 3, bytestream -5 [in#0/rtsp @ 0x55591f086e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55591f086e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T20:57:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T20:57:16+09:00", "pid": 23177, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.078, "name": "ffmpeg", "pid": 23177, "rss_mb": 209.078, "state": "S (sleeping)", "threads": 106}, "restart_count": 8, "restart_count_per_hour": 1, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.039, "sample_count": 2539, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 215420, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:57:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.918, "last_frame_pushed_at": "2026-07-01T20:57:13+09:00", "last_frame_read_age_sec": 4.988, "last_frame_read_at": "2026-07-01T20:57:13+09:00", "mjpeg": {"age_sec": 4.908, "exists": true, "mtime": "2026-07-01T20:57:13.114633+09:00", "mtime_epoch": 1782907033.114633, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97020}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T20:57:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 967.16, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.003, "pushed": 2.003, "read": 1.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.613, "count": 60, "max_ms": 4.436, "p95_ms": 3.825}, "gstreamer_push": {"avg_ms": 1.629, "count": 60, "max_ms": 5.255, "p95_ms": 3.674}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.363, "count": 60, "max_ms": 76.689, "p95_ms": 73.998}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.518, "count": 60, "max_ms": 75.556, "p95_ms": 74.46}, "yolo11_640_fire_11class": {"avg_ms": 53.345, "count": 60, "max_ms": 74.434, "p95_ms": 73.059}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.239, "count": 60, "max_ms": 84.823, "p95_ms": 83.818}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.082, "count": 60, "max_ms": 97.866, "p95_ms": 96.942}, "read_interval": {"avg_ms": 424.841, "count": 59, "max_ms": 10769.353, "p95_ms": 118.046}, "triton_wait": {"avg_ms": 57.491, "count": 60, "max_ms": 84.835, "p95_ms": 83.83}, "visualize": {"avg_ms": 3.674, "count": 60, "max_ms": 4.258, "p95_ms": 3.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1804}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4025.117, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55fea213c800] error while decoding MB 81 29, bytestream -5 [in#0/rtsp @ 0x55fea20fae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fea20fae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:59:58+09:00", "last_error_summary": "[h264 @ 0x55fea213c800] error while decoding MB 81 29, bytestream -5 [in#0/rtsp @ 0x55fea20fae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fea20fae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T20:57:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.459, "last_started_at": "2026-07-01T20:57:16+09:00", "pid": 23177, "process": {"exists": false, "pid": 23177}, "restart_count": 8, "restart_count_per_hour": 1, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.117, "sample_count": 2217, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 216582, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:59:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T20:59:58+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T20:59:58+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T20:59:58.418206+09:00", "mtime_epoch": 1782907198.418206, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93119}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T20:59:58+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 967.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.742, "pushed": 0.742, "read": 0.799}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.623, "count": 13, "max_ms": 3.69, "p95_ms": 3.69}, "gstreamer_push": {"avg_ms": 1.433, "count": 13, "max_ms": 1.979, "p95_ms": 1.979}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.553, "count": 13, "max_ms": 59.754, "p95_ms": 59.754}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.392, "count": 13, "max_ms": 67.633, "p95_ms": 67.633}, "yolo11_640_fire_11class": {"avg_ms": 52.717, "count": 13, "max_ms": 66.594, "p95_ms": 66.594}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.756, "count": 13, "max_ms": 71.95, "p95_ms": 71.95}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.612, "count": 13, "max_ms": 84.585, "p95_ms": 84.585}, "read_interval": {"avg_ms": 2364.479, "count": 14, "max_ms": 16469.657, "p95_ms": 16469.657}, "triton_wait": {"avg_ms": 55.925, "count": 13, "max_ms": 71.962, "p95_ms": 71.962}, "visualize": {"avg_ms": 3.669, "count": 13, "max_ms": 3.732, "p95_ms": 3.732}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.519, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1804}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4024.922, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T20:59:58+09:00", "last_error_summary": "[h264 @ 0x55fea213c800] error while decoding MB 81 29, bytestream -5 [in#0/rtsp @ 0x55fea20fae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fea20fae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:59:58+09:00", "last_error_summary": "[h264 @ 0x55fea213c800] error while decoding MB 81 29, bytestream -5 [in#0/rtsp @ 0x55fea20fae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fea20fae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T20:59:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.48, "last_started_at": "2026-07-01T20:57:16+09:00", "pid": 23177, "process": {"exists": false, "pid": 23177}, "restart_count": 8, "restart_count_per_hour": 1, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.117, "sample_count": 2218, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 216583, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:59:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T20:59:58+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-01T20:59:58+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-01T20:59:58.418206+09:00", "mtime_epoch": 1782907198.418206, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93119}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T20:59:58+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 967.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.798, "pushed": 0.798, "read": 0.798}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.636, "count": 14, "max_ms": 3.81, "p95_ms": 3.81}, "gstreamer_push": {"avg_ms": 1.447, "count": 14, "max_ms": 1.979, "p95_ms": 1.979}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.898, "count": 14, "max_ms": 59.754, "p95_ms": 59.754}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.636, "count": 14, "max_ms": 67.633, "p95_ms": 67.633}, "yolo11_640_fire_11class": {"avg_ms": 53.047, "count": 14, "max_ms": 66.594, "p95_ms": 66.594}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.965, "count": 14, "max_ms": 71.95, "p95_ms": 71.95}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.089, "count": 14, "max_ms": 84.585, "p95_ms": 84.585}, "read_interval": {"avg_ms": 2364.479, "count": 14, "max_ms": 16469.657, "p95_ms": 16469.657}, "triton_wait": {"avg_ms": 56.122, "count": 14, "max_ms": 71.962, "p95_ms": 71.962}, "visualize": {"avg_ms": 3.93, "count": 14, "max_ms": 7.323, "p95_ms": 7.323}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.539, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1804}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4024.812, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:59:58+09:00", "last_error_summary": "[h264 @ 0x55fea213c800] error while decoding MB 81 29, bytestream -5 [in#0/rtsp @ 0x55fea20fae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fea20fae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T20:59:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.487, "last_started_at": "2026-07-01T20:57:16+09:00", "pid": 23177, "process": {"exists": false, "pid": 23177}, "restart_count": 8, "restart_count_per_hour": 1, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.117, "sample_count": 2218, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 216583, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:59:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T20:59:58+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-01T20:59:58+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T20:59:58.418206+09:00", "mtime_epoch": 1782907198.418206, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93119}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T20:59:58+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 967.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.798, "pushed": 0.798, "read": 0.798}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.636, "count": 14, "max_ms": 3.81, "p95_ms": 3.81}, "gstreamer_push": {"avg_ms": 1.447, "count": 14, "max_ms": 1.979, "p95_ms": 1.979}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.898, "count": 14, "max_ms": 59.754, "p95_ms": 59.754}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.636, "count": 14, "max_ms": 67.633, "p95_ms": 67.633}, "yolo11_640_fire_11class": {"avg_ms": 53.047, "count": 14, "max_ms": 66.594, "p95_ms": 66.594}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.965, "count": 14, "max_ms": 71.95, "p95_ms": 71.95}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.089, "count": 14, "max_ms": 84.585, "p95_ms": 84.585}, "read_interval": {"avg_ms": 2364.479, "count": 14, "max_ms": 16469.657, "p95_ms": 16469.657}, "triton_wait": {"avg_ms": 56.122, "count": 14, "max_ms": 71.962, "p95_ms": 71.962}, "visualize": {"avg_ms": 3.93, "count": 14, "max_ms": 7.323, "p95_ms": 7.323}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.547, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1804}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4204.016, "limit_mb": 8796093022207.996}, "detail": {"pid": 23329, "restart_count": 9, "start_count": 10}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T20:59:58+09:00", "last_error_summary": "[h264 @ 0x55fea213c800] error while decoding MB 81 29, bytestream -5 [in#0/rtsp @ 0x55fea20fae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55fea20fae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:00:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:00:01+09:00", "pid": 23329, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.195, "name": "ffmpeg", "pid": 23329, "rss_mb": 209.195, "state": "S (sleeping)", "threads": 106}, "restart_count": 9, "restart_count_per_hour": 2, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.137, "sample_count": 2176, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 216583, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T20:59:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.613, "last_frame_pushed_at": "2026-07-01T20:59:58+09:00", "last_frame_read_age_sec": 4.691, "last_frame_read_at": "2026-07-01T20:59:58+09:00", "mjpeg": {"age_sec": 4.598, "exists": true, "mtime": "2026-07-01T20:59:58.510208+09:00", "mtime_epoch": 1782907198.5102081, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92799}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:00:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 967.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.632, "pushed": 0.632, "read": 0.632}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.636, "count": 14, "max_ms": 3.81, "p95_ms": 3.81}, "gstreamer_push": {"avg_ms": 1.447, "count": 14, "max_ms": 1.979, "p95_ms": 1.979}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.898, "count": 14, "max_ms": 59.754, "p95_ms": 59.754}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.636, "count": 14, "max_ms": 67.633, "p95_ms": 67.633}, "yolo11_640_fire_11class": {"avg_ms": 53.047, "count": 14, "max_ms": 66.594, "p95_ms": 66.594}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.965, "count": 14, "max_ms": 71.95, "p95_ms": 71.95}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.089, "count": 14, "max_ms": 84.585, "p95_ms": 84.585}, "read_interval": {"avg_ms": 2364.479, "count": 14, "max_ms": 16469.657, "p95_ms": 16469.657}, "triton_wait": {"avg_ms": 56.122, "count": 14, "max_ms": 71.962, "p95_ms": 71.962}, "visualize": {"avg_ms": 3.93, "count": 14, "max_ms": 7.323, "p95_ms": 7.323}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.146, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4036.883, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b37bfce2c0] error while decoding MB 113 33, bytestream -11 [in#0/rtsp @ 0x55b37ba3fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b37ba3fe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:06:10+09:00", "last_error_summary": "[h264 @ 0x55b37bfce2c0] error while decoding MB 113 33, bytestream -11 [in#0/rtsp @ 0x55b37ba3fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b37ba3fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:00:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 369.38, "last_started_at": "2026-07-01T21:00:01+09:00", "pid": 23329, "process": {"exists": false, "pid": 23329}, "restart_count": 9, "restart_count_per_hour": 2, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.44, "sample_count": 2528, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 219725, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:06:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T21:06:10+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-01T21:06:10+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-01T21:06:10.426084+09:00", "mtime_epoch": 1782907570.4260843, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:06:10+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 968.98, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.268, "pushed": 3.268, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.677, "count": 99, "max_ms": 5.167, "p95_ms": 4.717}, "gstreamer_push": {"avg_ms": 1.658, "count": 98, "max_ms": 6.089, "p95_ms": 3.761}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.266, "count": 99, "max_ms": 79.375, "p95_ms": 76.311}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.235, "count": 99, "max_ms": 79.969, "p95_ms": 77.735}, "yolo11_640_fire_11class": {"avg_ms": 69.872, "count": 99, "max_ms": 79.387, "p95_ms": 76.373}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.818, "count": 99, "max_ms": 88.436, "p95_ms": 86.496}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.688, "count": 98, "max_ms": 100.642, "p95_ms": 99.55}, "read_interval": {"avg_ms": 303.377, "count": 99, "max_ms": 10163.696, "p95_ms": 117.57}, "triton_wait": {"avg_ms": 77.851, "count": 99, "max_ms": 88.448, "p95_ms": 86.509}, "visualize": {"avg_ms": 3.691, "count": 99, "max_ms": 4.631, "p95_ms": 4.38}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4036.492, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:06:10+09:00", "last_error_summary": "[h264 @ 0x55b37bfce2c0] error while decoding MB 113 33, bytestream -11 [in#0/rtsp @ 0x55b37ba3fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b37ba3fe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:06:10+09:00", "last_error_summary": "[h264 @ 0x55b37bfce2c0] error while decoding MB 113 33, bytestream -11 [in#0/rtsp @ 0x55b37ba3fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b37ba3fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:06:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 369.402, "last_started_at": "2026-07-01T21:00:01+09:00", "pid": 23329, "process": {"exists": false, "pid": 23329}, "restart_count": 9, "restart_count_per_hour": 2, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.44, "sample_count": 2529, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 219725, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:06:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T21:06:10+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-01T21:06:10+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-01T21:06:10.426084+09:00", "mtime_epoch": 1782907570.4260843, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:06:10+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 968.98, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.31, "pushed": 3.31, "read": 3.276}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.677, "count": 99, "max_ms": 5.167, "p95_ms": 4.717}, "gstreamer_push": {"avg_ms": 1.655, "count": 99, "max_ms": 6.089, "p95_ms": 3.761}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.266, "count": 99, "max_ms": 79.375, "p95_ms": 76.311}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.235, "count": 99, "max_ms": 79.969, "p95_ms": 77.735}, "yolo11_640_fire_11class": {"avg_ms": 69.872, "count": 99, "max_ms": 79.387, "p95_ms": 76.373}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.818, "count": 99, "max_ms": 88.436, "p95_ms": 86.496}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.506, "count": 99, "max_ms": 100.642, "p95_ms": 99.55}, "read_interval": {"avg_ms": 305.339, "count": 98, "max_ms": 10163.696, "p95_ms": 117.57}, "triton_wait": {"avg_ms": 77.851, "count": 99, "max_ms": 88.448, "p95_ms": 86.509}, "visualize": {"avg_ms": 3.691, "count": 99, "max_ms": 4.631, "p95_ms": 4.38}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4036.512, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:06:10+09:00", "last_error_summary": "[h264 @ 0x55b37bfce2c0] error while decoding MB 113 33, bytestream -11 [in#0/rtsp @ 0x55b37ba3fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b37ba3fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:06:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 369.41, "last_started_at": "2026-07-01T21:00:01+09:00", "pid": 23329, "process": {"exists": false, "pid": 23329}, "restart_count": 9, "restart_count_per_hour": 2, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.44, "sample_count": 2529, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 219725, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:06:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-01T21:06:10+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T21:06:10+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-01T21:06:10.426084+09:00", "mtime_epoch": 1782907570.4260843, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:06:10+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 968.98, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.309, "pushed": 3.309, "read": 3.275}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.677, "count": 99, "max_ms": 5.167, "p95_ms": 4.717}, "gstreamer_push": {"avg_ms": 1.655, "count": 99, "max_ms": 6.089, "p95_ms": 3.761}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.266, "count": 99, "max_ms": 79.375, "p95_ms": 76.311}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.235, "count": 99, "max_ms": 79.969, "p95_ms": 77.735}, "yolo11_640_fire_11class": {"avg_ms": 69.872, "count": 99, "max_ms": 79.387, "p95_ms": 76.373}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.818, "count": 99, "max_ms": 88.436, "p95_ms": 86.496}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.506, "count": 99, "max_ms": 100.642, "p95_ms": 99.55}, "read_interval": {"avg_ms": 305.339, "count": 98, "max_ms": 10163.696, "p95_ms": 117.57}, "triton_wait": {"avg_ms": 77.851, "count": 99, "max_ms": 88.448, "p95_ms": 86.509}, "visualize": {"avg_ms": 3.691, "count": 99, "max_ms": 4.631, "p95_ms": 4.38}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1831}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4059.695, "limit_mb": 8796093022207.996}, "detail": {"pid": 23481, "restart_count": 10, "start_count": 11}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:06:10+09:00", "last_error_summary": "[h264 @ 0x55b37bfce2c0] error while decoding MB 113 33, bytestream -11 [in#0/rtsp @ 0x55b37ba3fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b37ba3fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:06:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:06:13+09:00", "pid": 23481, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.285, "name": "ffmpeg", "pid": 23481, "rss_mb": 58.285, "state": "S (sleeping)", "threads": 1}, "restart_count": 10, "restart_count_per_hour": 3, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.48, "sample_count": 2486, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 219725, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:06:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.839, "last_frame_pushed_at": "2026-07-01T21:06:10+09:00", "last_frame_read_age_sec": 4.913, "last_frame_read_at": "2026-07-01T21:06:10+09:00", "mjpeg": {"age_sec": 4.823, "exists": true, "mtime": "2026-07-01T21:06:10.514086+09:00", "mtime_epoch": 1782907570.514086, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92979}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:06:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 968.98, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.868, "pushed": 1.868, "read": 1.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.63, "count": 56, "max_ms": 5.167, "p95_ms": 3.692}, "gstreamer_push": {"avg_ms": 1.569, "count": 56, "max_ms": 3.877, "p95_ms": 3.625}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.422, "count": 56, "max_ms": 78.922, "p95_ms": 76.466}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.598, "count": 56, "max_ms": 78.176, "p95_ms": 77.873}, "yolo11_640_fire_11class": {"avg_ms": 68.122, "count": 56, "max_ms": 78.912, "p95_ms": 77.049}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.674, "count": 56, "max_ms": 88.436, "p95_ms": 87.142}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.259, "count": 56, "max_ms": 100.642, "p95_ms": 99.55}, "read_interval": {"avg_ms": 457.57, "count": 55, "max_ms": 10163.696, "p95_ms": 116.906}, "triton_wait": {"avg_ms": 75.723, "count": 56, "max_ms": 88.448, "p95_ms": 87.156}, "visualize": {"avg_ms": 3.658, "count": 56, "max_ms": 3.971, "p95_ms": 3.873}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4088.688, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b2b9281040] left block unavailable for requested intra mode [h264 @ 0x55b2b9281040] error while decoding MB 0 17, bytestream 8630 [h264 @ 0x55b2b891a3c0] error while decoding MB 69 24, bytestream -24 [in#0/rtsp @ 0x55b2b88d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2b88d6e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 157, "h264_decode_error_count": 172, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:08:37+09:00", "last_error_summary": "[h264 @ 0x55b2b9281040] left block unavailable for requested intra mode [h264 @ 0x55b2b9281040] error while decoding MB 0 17, bytestream 8630 [h264 @ 0x55b2b891a3c0] error while decoding MB 69 24, bytestream -24 [in#0/rtsp @ 0x55b2b88d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2b88d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T20:57:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 684.732, "last_started_at": "2026-07-01T20:57:12+09:00", "pid": 23070, "process": {"exists": false, "pid": 23070}, "restart_count": 181, "restart_count_per_hour": 17, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 182}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.663, "sample_count": 2634, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 102524, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:08:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-01T21:08:37+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-01T21:08:37+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T21:08:37.605161+09:00", "mtime_epoch": 1782907717.6051612, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114014}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:08:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 910.902, "name": "celery", "pid": 237, "rss_mb": 909.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.405, "pushed": 3.405, "read": 3.405}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.636, "count": 103, "max_ms": 2.144, "p95_ms": 2.089}, "gstreamer_push": {"avg_ms": 1.605, "count": 102, "max_ms": 6.239, "p95_ms": 3.756}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.39, "count": 103, "max_ms": 74.862, "p95_ms": 67.897}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.746, "count": 103, "max_ms": 77.778, "p95_ms": 76.645}, "yolo11_640_fire_11class": {"avg_ms": 70.917, "count": 103, "max_ms": 76.662, "p95_ms": 75.518}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.099, "count": 103, "max_ms": 87.343, "p95_ms": 86.632}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.948, "count": 102, "max_ms": 97.971, "p95_ms": 97.166}, "read_interval": {"avg_ms": 294.082, "count": 102, "max_ms": 19156.781, "p95_ms": 115.222}, "triton_wait": {"avg_ms": 80.171, "count": 103, "max_ms": 87.359, "p95_ms": 86.646}, "visualize": {"avg_ms": 3.706, "count": 103, "max_ms": 4.452, "p95_ms": 4.02}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4088.5, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:08:37+09:00", "last_error_summary": "[h264 @ 0x55b2b9281040] left block unavailable for requested intra mode [h264 @ 0x55b2b9281040] error while decoding MB 0 17, bytestream 8630 [h264 @ 0x55b2b891a3c0] error while decoding MB 69 24, bytestream -24 [in#0/rtsp @ 0x55b2b88d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2b88d6e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 157, "h264_decode_error_count": 172, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:08:37+09:00", "last_error_summary": "[h264 @ 0x55b2b9281040] left block unavailable for requested intra mode [h264 @ 0x55b2b9281040] error while decoding MB 0 17, bytestream 8630 [h264 @ 0x55b2b891a3c0] error while decoding MB 69 24, bytestream -24 [in#0/rtsp @ 0x55b2b88d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2b88d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:08:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 684.75, "last_started_at": "2026-07-01T20:57:12+09:00", "pid": 23070, "process": {"exists": false, "pid": 23070}, "restart_count": 181, "restart_count_per_hour": 17, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 182}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.663, "sample_count": 2635, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 102524, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:08:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T21:08:37+09:00", "last_frame_read_age_sec": 0.074, "last_frame_read_at": "2026-07-01T21:08:37+09:00", "mjpeg": {"age_sec": 0.067, "exists": true, "mtime": "2026-07-01T21:08:37.605161+09:00", "mtime_epoch": 1782907717.6051612, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114014}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:08:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 910.902, "name": "celery", "pid": 237, "rss_mb": 909.773, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.436, "pushed": 3.436, "read": 3.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.636, "count": 103, "max_ms": 2.144, "p95_ms": 2.089}, "gstreamer_push": {"avg_ms": 1.603, "count": 103, "max_ms": 6.239, "p95_ms": 3.756}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.39, "count": 103, "max_ms": 74.862, "p95_ms": 67.897}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.746, "count": 103, "max_ms": 77.778, "p95_ms": 76.645}, "yolo11_640_fire_11class": {"avg_ms": 70.917, "count": 103, "max_ms": 76.662, "p95_ms": 75.518}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.099, "count": 103, "max_ms": 87.343, "p95_ms": 86.632}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.7, "count": 103, "max_ms": 97.971, "p95_ms": 97.166}, "read_interval": {"avg_ms": 294.082, "count": 102, "max_ms": 19156.781, "p95_ms": 115.222}, "triton_wait": {"avg_ms": 80.171, "count": 103, "max_ms": 87.359, "p95_ms": 86.646}, "visualize": {"avg_ms": 3.706, "count": 103, "max_ms": 4.452, "p95_ms": 4.02}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4088.543, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 157, "h264_decode_error_count": 172, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:08:37+09:00", "last_error_summary": "[h264 @ 0x55b2b9281040] left block unavailable for requested intra mode [h264 @ 0x55b2b9281040] error while decoding MB 0 17, bytestream 8630 [h264 @ 0x55b2b891a3c0] error while decoding MB 69 24, bytestream -24 [in#0/rtsp @ 0x55b2b88d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2b88d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:08:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 684.758, "last_started_at": "2026-07-01T20:57:12+09:00", "pid": 23070, "process": {"exists": false, "pid": 23070}, "restart_count": 181, "restart_count_per_hour": 17, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 182}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.663, "sample_count": 2635, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 102524, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:08:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T21:08:37+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-01T21:08:37+09:00", "mjpeg": {"age_sec": 0.075, "exists": true, "mtime": "2026-07-01T21:08:37.605161+09:00", "mtime_epoch": 1782907717.6051612, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114014}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:08:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 910.902, "name": "celery", "pid": 237, "rss_mb": 909.773, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.435, "pushed": 3.435, "read": 3.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.636, "count": 103, "max_ms": 2.144, "p95_ms": 2.089}, "gstreamer_push": {"avg_ms": 1.603, "count": 103, "max_ms": 6.239, "p95_ms": 3.756}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.39, "count": 103, "max_ms": 74.862, "p95_ms": 67.897}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.746, "count": 103, "max_ms": 77.778, "p95_ms": 76.645}, "yolo11_640_fire_11class": {"avg_ms": 70.917, "count": 103, "max_ms": 76.662, "p95_ms": 75.518}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.099, "count": 103, "max_ms": 87.343, "p95_ms": 86.632}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.7, "count": 103, "max_ms": 97.971, "p95_ms": 97.166}, "read_interval": {"avg_ms": 294.082, "count": 102, "max_ms": 19156.781, "p95_ms": 115.222}, "triton_wait": {"avg_ms": 80.171, "count": 103, "max_ms": 87.359, "p95_ms": 86.646}, "visualize": {"avg_ms": 3.706, "count": 103, "max_ms": 4.452, "p95_ms": 4.02}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4195.914, "limit_mb": 8796093022207.996}, "detail": {"pid": 23633, "restart_count": 182, "start_count": 183}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 157, "h264_decode_error_count": 172, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:08:37+09:00", "last_error_summary": "[h264 @ 0x55b2b9281040] left block unavailable for requested intra mode [h264 @ 0x55b2b9281040] error while decoding MB 0 17, bytestream 8630 [h264 @ 0x55b2b891a3c0] error while decoding MB 69 24, bytestream -24 [in#0/rtsp @ 0x55b2b88d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2b88d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:08:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:08:40+09:00", "pid": 23633, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.414, "name": "ffmpeg", "pid": 23633, "rss_mb": 139.414, "state": "S (sleeping)", "threads": 106}, "restart_count": 182, "restart_count_per_hour": 18, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 183}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.702, "sample_count": 2593, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 102524, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:08:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.707, "last_frame_pushed_at": "2026-07-01T21:08:37+09:00", "last_frame_read_age_sec": 4.775, "last_frame_read_at": "2026-07-01T21:08:37+09:00", "mjpeg": {"age_sec": 4.692, "exists": true, "mtime": "2026-07-01T21:08:37.681163+09:00", "mtime_epoch": 1782907717.6811628, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114813}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:08:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 910.902, "name": "celery", "pid": 237, "rss_mb": 909.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.035, "pushed": 2.035, "read": 2.002}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.542, "count": 61, "max_ms": 1.968, "p95_ms": 1.609}, "gstreamer_push": {"avg_ms": 1.631, "count": 61, "max_ms": 6.239, "p95_ms": 3.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.871, "count": 61, "max_ms": 74.862, "p95_ms": 68.182}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.58, "count": 61, "max_ms": 77.778, "p95_ms": 76.65}, "yolo11_640_fire_11class": {"avg_ms": 68.962, "count": 61, "max_ms": 76.662, "p95_ms": 75.713}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.912, "count": 61, "max_ms": 87.343, "p95_ms": 86.598}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.453, "count": 61, "max_ms": 97.666, "p95_ms": 96.921}, "read_interval": {"avg_ms": 421.566, "count": 60, "max_ms": 19156.781, "p95_ms": 116.46}, "triton_wait": {"avg_ms": 77.024, "count": 61, "max_ms": 87.359, "p95_ms": 86.611}, "visualize": {"avg_ms": 3.662, "count": 61, "max_ms": 4.027, "p95_ms": 3.766}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1841}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4025.0, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x565306f9f200] missing picture in access unit with size 95 [h264 @ 0x565307022180] no frame! [in#0/rtsp @ 0x565306f9ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565306f9ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565306f9f840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:09:05+09:00", "last_error_summary": "[NULL @ 0x565306f9f200] missing picture in access unit with size 95 [h264 @ 0x565307022180] no frame! [in#0/rtsp @ 0x565306f9ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565306f9ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565306f9f840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:06:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 172.147, "last_started_at": "2026-07-01T21:06:13+09:00", "pid": 23481, "process": {"exists": false, "pid": 23481}, "restart_count": 10, "restart_count_per_hour": 3, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.623, "sample_count": 2208, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220973, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:09:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.092, "last_frame_pushed_at": "2026-07-01T21:09:05+09:00", "last_frame_read_age_sec": 0.076, "last_frame_read_at": "2026-07-01T21:09:05+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-01T21:09:05.405741+09:00", "mtime_epoch": 1782907745.4057407, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100890}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:09:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 969.496, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.676, "pushed": 0.676, "read": 0.732}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.632, "count": 12, "max_ms": 3.923, "p95_ms": 3.923}, "gstreamer_push": {"avg_ms": 1.343, "count": 12, "max_ms": 1.413, "p95_ms": 1.413}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.201, "count": 12, "max_ms": 75.562, "p95_ms": 75.562}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.526, "count": 12, "max_ms": 77.618, "p95_ms": 77.618}, "yolo11_640_fire_11class": {"avg_ms": 71.575, "count": 12, "max_ms": 75.571, "p95_ms": 75.571}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.206, "count": 12, "max_ms": 85.296, "p95_ms": 85.296}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.886, "count": 12, "max_ms": 98.126, "p95_ms": 98.126}, "read_interval": {"avg_ms": 2929.109, "count": 13, "max_ms": 20396.03, "p95_ms": 20396.03}, "triton_wait": {"avg_ms": 82.328, "count": 12, "max_ms": 85.312, "p95_ms": 85.312}, "visualize": {"avg_ms": 3.709, "count": 12, "max_ms": 3.764, "p95_ms": 3.764}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.759, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1841}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4024.871, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:09:05+09:00", "last_error_summary": "[NULL @ 0x565306f9f200] missing picture in access unit with size 95 [h264 @ 0x565307022180] no frame! [in#0/rtsp @ 0x565306f9ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565306f9ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565306f9f840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:09:05+09:00", "last_error_summary": "[NULL @ 0x565306f9f200] missing picture in access unit with size 95 [h264 @ 0x565307022180] no frame! [in#0/rtsp @ 0x565306f9ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565306f9ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565306f9f840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:09:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 172.183, "last_started_at": "2026-07-01T21:06:13+09:00", "pid": 23481, "process": {"exists": false, "pid": 23481}, "restart_count": 10, "restart_count_per_hour": 3, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.626, "sample_count": 2208, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220974, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:09:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T21:09:05+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T21:09:05+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-01T21:09:05.405741+09:00", "mtime_epoch": 1782907745.4057407, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100890}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:09:05+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 969.496, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.731, "pushed": 0.731, "read": 0.731}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.611, "count": 13, "max_ms": 3.923, "p95_ms": 3.923}, "gstreamer_push": {"avg_ms": 1.339, "count": 13, "max_ms": 1.413, "p95_ms": 1.413}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.882, "count": 13, "max_ms": 84.05, "p95_ms": 84.05}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.365, "count": 13, "max_ms": 85.436, "p95_ms": 85.436}, "yolo11_640_fire_11class": {"avg_ms": 72.641, "count": 13, "max_ms": 85.424, "p95_ms": 85.424}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.11, "count": 13, "max_ms": 92.956, "p95_ms": 92.956}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.685, "count": 13, "max_ms": 105.271, "p95_ms": 105.271}, "read_interval": {"avg_ms": 2929.109, "count": 13, "max_ms": 20396.03, "p95_ms": 20396.03}, "triton_wait": {"avg_ms": 83.147, "count": 13, "max_ms": 92.967, "p95_ms": 92.967}, "visualize": {"avg_ms": 3.712, "count": 13, "max_ms": 3.764, "p95_ms": 3.764}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.795, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1841}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4024.543, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:09:05+09:00", "last_error_summary": "[NULL @ 0x565306f9f200] missing picture in access unit with size 95 [h264 @ 0x565307022180] no frame! [in#0/rtsp @ 0x565306f9ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565306f9ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565306f9f840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:09:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 172.191, "last_started_at": "2026-07-01T21:06:13+09:00", "pid": 23481, "process": {"exists": false, "pid": 23481}, "restart_count": 10, "restart_count_per_hour": 3, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.626, "sample_count": 2208, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220974, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:09:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T21:09:05+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-01T21:09:05+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-01T21:09:05.405741+09:00", "mtime_epoch": 1782907745.4057407, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100890}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:09:05+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 969.496, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.73, "pushed": 0.73, "read": 0.73}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.611, "count": 13, "max_ms": 3.923, "p95_ms": 3.923}, "gstreamer_push": {"avg_ms": 1.339, "count": 13, "max_ms": 1.413, "p95_ms": 1.413}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.882, "count": 13, "max_ms": 84.05, "p95_ms": 84.05}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.365, "count": 13, "max_ms": 85.436, "p95_ms": 85.436}, "yolo11_640_fire_11class": {"avg_ms": 72.641, "count": 13, "max_ms": 85.424, "p95_ms": 85.424}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.11, "count": 13, "max_ms": 92.956, "p95_ms": 92.956}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.685, "count": 13, "max_ms": 105.271, "p95_ms": 105.271}, "read_interval": {"avg_ms": 2929.109, "count": 13, "max_ms": 20396.03, "p95_ms": 20396.03}, "triton_wait": {"avg_ms": 83.147, "count": 13, "max_ms": 92.967, "p95_ms": 92.967}, "visualize": {"avg_ms": 3.712, "count": 13, "max_ms": 3.764, "p95_ms": 3.764}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.803, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1841}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4148.566, "limit_mb": 8796093022207.996}, "detail": {"pid": 23740, "restart_count": 11, "start_count": 12}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:09:05+09:00", "last_error_summary": "[NULL @ 0x565306f9f200] missing picture in access unit with size 95 [h264 @ 0x565307022180] no frame! [in#0/rtsp @ 0x565306f9ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565306f9ae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565306f9f840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:09:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:09:07+09:00", "pid": 23740, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.512, "name": "ffmpeg", "pid": 23740, "rss_mb": 156.512, "state": "R (running)", "threads": 49}, "restart_count": 11, "restart_count_per_hour": 4, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.623, "sample_count": 2171, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220974, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:09:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.225, "last_frame_pushed_at": "2026-07-01T21:09:05+09:00", "last_frame_read_age_sec": 4.333, "last_frame_read_at": "2026-07-01T21:09:05+09:00", "mjpeg": {"age_sec": 4.21, "exists": true, "mtime": "2026-07-01T21:09:05.529743+09:00", "mtime_epoch": 1782907745.5297434, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101592}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:09:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 969.496, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.59, "pushed": 0.59, "read": 0.59}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.611, "count": 13, "max_ms": 3.923, "p95_ms": 3.923}, "gstreamer_push": {"avg_ms": 1.339, "count": 13, "max_ms": 1.413, "p95_ms": 1.413}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.882, "count": 13, "max_ms": 84.05, "p95_ms": 84.05}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.365, "count": 13, "max_ms": 85.436, "p95_ms": 85.436}, "yolo11_640_fire_11class": {"avg_ms": 72.641, "count": 13, "max_ms": 85.424, "p95_ms": 85.424}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.11, "count": 13, "max_ms": 92.956, "p95_ms": 92.956}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.685, "count": 13, "max_ms": 105.271, "p95_ms": 105.271}, "read_interval": {"avg_ms": 2929.109, "count": 13, "max_ms": 20396.03, "p95_ms": 20396.03}, "triton_wait": {"avg_ms": 83.147, "count": 13, "max_ms": 92.967, "p95_ms": 92.967}, "visualize": {"avg_ms": 3.712, "count": 13, "max_ms": 3.764, "p95_ms": 3.764}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.015, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4040.0, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x563a654d7200] missing picture in access unit with size 18 [h264 @ 0x563a65514a00] no frame! [in#0/rtsp @ 0x563a654d2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a654d2e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563a654d7840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:12:25+09:00", "last_error_summary": "[NULL @ 0x563a654d7200] missing picture in access unit with size 18 [h264 @ 0x563a65514a00] no frame! [in#0/rtsp @ 0x563a654d2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a654d2e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563a654d7840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:12:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 198.206, "last_started_at": "2026-07-01T21:09:07+09:00", "pid": 23740, "process": {"exists": false, "pid": 23740}, "restart_count": 11, "restart_count_per_hour": 4, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.852, "sample_count": 2150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 222506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:12:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T21:12:25+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-01T21:12:25+09:00", "mjpeg": {"age_sec": 0.072, "exists": true, "mtime": "2026-07-01T21:12:25.872829+09:00", "mtime_epoch": 1782907945.8728292, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89762}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:12:25+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 975.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 33, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.137, "pushed": 1.137, "read": 1.104}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.637, "count": 34, "max_ms": 4.008, "p95_ms": 3.95}, "gstreamer_push": {"avg_ms": 1.457, "count": 34, "max_ms": 3.986, "p95_ms": 1.539}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.704, "count": 34, "max_ms": 76.197, "p95_ms": 75.467}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.393, "count": 34, "max_ms": 80.047, "p95_ms": 79.23}, "yolo11_640_fire_11class": {"avg_ms": 63.859, "count": 34, "max_ms": 77.881, "p95_ms": 77.231}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.242, "count": 34, "max_ms": 86.823, "p95_ms": 86.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.027, "count": 34, "max_ms": 100.412, "p95_ms": 99.338}, "read_interval": {"avg_ms": 906.981, "count": 33, "max_ms": 26638.582, "p95_ms": 167.768}, "triton_wait": {"avg_ms": 71.285, "count": 34, "max_ms": 86.837, "p95_ms": 86.767}, "visualize": {"avg_ms": 3.751, "count": 34, "max_ms": 4.759, "p95_ms": 3.946}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4039.969, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:12:25+09:00", "last_error_summary": "[NULL @ 0x563a654d7200] missing picture in access unit with size 18 [h264 @ 0x563a65514a00] no frame! [in#0/rtsp @ 0x563a654d2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a654d2e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563a654d7840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:12:25+09:00", "last_error_summary": "[NULL @ 0x563a654d7200] missing picture in access unit with size 18 [h264 @ 0x563a65514a00] no frame! [in#0/rtsp @ 0x563a654d2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a654d2e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563a654d7840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:12:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 198.218, "last_started_at": "2026-07-01T21:09:07+09:00", "pid": 23740, "process": {"exists": false, "pid": 23740}, "restart_count": 11, "restart_count_per_hour": 4, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.852, "sample_count": 2150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 222506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:12:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-01T21:12:25+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-01T21:12:25+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-01T21:12:25.872829+09:00", "mtime_epoch": 1782907945.8728292, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89762}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:12:25+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 975.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 33, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.137, "pushed": 1.137, "read": 1.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.637, "count": 34, "max_ms": 4.008, "p95_ms": 3.95}, "gstreamer_push": {"avg_ms": 1.457, "count": 34, "max_ms": 3.986, "p95_ms": 1.539}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.704, "count": 34, "max_ms": 76.197, "p95_ms": 75.467}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.393, "count": 34, "max_ms": 80.047, "p95_ms": 79.23}, "yolo11_640_fire_11class": {"avg_ms": 63.859, "count": 34, "max_ms": 77.881, "p95_ms": 77.231}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.242, "count": 34, "max_ms": 86.823, "p95_ms": 86.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.027, "count": 34, "max_ms": 100.412, "p95_ms": 99.338}, "read_interval": {"avg_ms": 906.981, "count": 33, "max_ms": 26638.582, "p95_ms": 167.768}, "triton_wait": {"avg_ms": 71.285, "count": 34, "max_ms": 86.837, "p95_ms": 86.767}, "visualize": {"avg_ms": 3.751, "count": 34, "max_ms": 4.759, "p95_ms": 3.946}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4039.824, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:12:25+09:00", "last_error_summary": "[NULL @ 0x563a654d7200] missing picture in access unit with size 18 [h264 @ 0x563a65514a00] no frame! [in#0/rtsp @ 0x563a654d2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a654d2e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563a654d7840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:12:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 198.226, "last_started_at": "2026-07-01T21:09:07+09:00", "pid": 23740, "process": {"exists": false, "pid": 23740}, "restart_count": 11, "restart_count_per_hour": 4, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.852, "sample_count": 2150, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 222506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:12:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.026, "last_frame_pushed_at": "2026-07-01T21:12:25+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-01T21:12:25+09:00", "mjpeg": {"age_sec": 0.008, "exists": true, "mtime": "2026-07-01T21:12:25.956830+09:00", "mtime_epoch": 1782907945.9568303, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90928}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:12:25+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 975.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 33, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.136, "pushed": 1.136, "read": 1.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.637, "count": 34, "max_ms": 4.008, "p95_ms": 3.95}, "gstreamer_push": {"avg_ms": 1.457, "count": 34, "max_ms": 3.986, "p95_ms": 1.539}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.704, "count": 34, "max_ms": 76.197, "p95_ms": 75.467}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.393, "count": 34, "max_ms": 80.047, "p95_ms": 79.23}, "yolo11_640_fire_11class": {"avg_ms": 63.859, "count": 34, "max_ms": 77.881, "p95_ms": 77.231}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.242, "count": 34, "max_ms": 86.823, "p95_ms": 86.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.027, "count": 34, "max_ms": 100.412, "p95_ms": 99.338}, "read_interval": {"avg_ms": 906.981, "count": 33, "max_ms": 26638.582, "p95_ms": 167.768}, "triton_wait": {"avg_ms": 71.285, "count": 34, "max_ms": 86.837, "p95_ms": 86.767}, "visualize": {"avg_ms": 3.751, "count": 34, "max_ms": 4.759, "p95_ms": 3.946}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1854}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4211.691, "limit_mb": 8796093022207.996}, "detail": {"pid": 23892, "restart_count": 12, "start_count": 13}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:12:25+09:00", "last_error_summary": "[NULL @ 0x563a654d7200] missing picture in access unit with size 18 [h264 @ 0x563a65514a00] no frame! [in#0/rtsp @ 0x563a654d2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563a654d2e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563a654d7840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:12:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:12:28+09:00", "pid": 23892, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.703, "name": "ffmpeg", "pid": 23892, "rss_mb": 206.703, "state": "S (sleeping)", "threads": 106}, "restart_count": 12, "restart_count_per_hour": 5, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.863, "sample_count": 2113, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 222506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:12:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.081, "last_frame_pushed_at": "2026-07-01T21:12:25+09:00", "last_frame_read_age_sec": 4.147, "last_frame_read_at": "2026-07-01T21:12:25+09:00", "mjpeg": {"age_sec": 4.063, "exists": true, "mtime": "2026-07-01T21:12:25.956830+09:00", "mtime_epoch": 1782907945.9568303, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90928}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:12:30+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 975.742, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.384, "pushed": 2.384, "read": 2.384}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.612, "count": 12, "max_ms": 4.008, "p95_ms": 4.008}, "gstreamer_push": {"avg_ms": 1.352, "count": 12, "max_ms": 1.531, "p95_ms": 1.531}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.638, "count": 12, "max_ms": 56.947, "p95_ms": 56.947}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.29, "count": 12, "max_ms": 57.284, "p95_ms": 57.284}, "yolo11_640_fire_11class": {"avg_ms": 50.228, "count": 12, "max_ms": 56.286, "p95_ms": 56.286}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.981, "count": 12, "max_ms": 58.999, "p95_ms": 58.999}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.761, "count": 12, "max_ms": 71.921, "p95_ms": 71.921}, "read_interval": {"avg_ms": 2293.77, "count": 12, "max_ms": 26638.582, "p95_ms": 26638.582}, "triton_wait": {"avg_ms": 53.078, "count": 12, "max_ms": 59.011, "p95_ms": 59.011}, "visualize": {"avg_ms": 3.753, "count": 12, "max_ms": 4.759, "p95_ms": 4.759}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.034, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1864}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4045.324, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55dbf3d4da40] error while decoding MB 101 18, bytestream -5 [in#0/rtsp @ 0x55dbf32e0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dbf32e0e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:15:18+09:00", "last_error_summary": "[h264 @ 0x55dbf3d4da40] error while decoding MB 101 18, bytestream -5 [in#0/rtsp @ 0x55dbf32e0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dbf32e0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:12:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 170.787, "last_started_at": "2026-07-01T21:12:28+09:00", "pid": 23892, "process": {"exists": false, "pid": 23892}, "restart_count": 12, "restart_count_per_hour": 5, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.059, "sample_count": 2135, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 223729, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:15:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T21:15:18+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-01T21:15:18+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T21:15:18.742952+09:00", "mtime_epoch": 1782908118.7429523, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:15:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 970.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.727, "pushed": 0.727, "read": 0.783}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.41, "count": 13, "max_ms": 6.058, "p95_ms": 6.058}, "gstreamer_push": {"avg_ms": 1.712, "count": 13, "max_ms": 3.746, "p95_ms": 3.746}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.268, "count": 13, "max_ms": 74.88, "p95_ms": 74.88}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.182, "count": 13, "max_ms": 75.723, "p95_ms": 75.723}, "yolo11_640_fire_11class": {"avg_ms": 71.819, "count": 13, "max_ms": 76.738, "p95_ms": 76.738}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.73, "count": 13, "max_ms": 87.764, "p95_ms": 87.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.193, "count": 13, "max_ms": 107.43, "p95_ms": 107.43}, "read_interval": {"avg_ms": 2414.923, "count": 14, "max_ms": 16408.244, "p95_ms": 16408.244}, "triton_wait": {"avg_ms": 80.747, "count": 13, "max_ms": 87.78, "p95_ms": 87.78}, "visualize": {"avg_ms": 5.603, "count": 13, "max_ms": 6.652, "p95_ms": 6.652}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.873, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1864}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4044.398, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:15:18+09:00", "last_error_summary": "[h264 @ 0x55dbf3d4da40] error while decoding MB 101 18, bytestream -5 [in#0/rtsp @ 0x55dbf32e0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dbf32e0e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:15:18+09:00", "last_error_summary": "[h264 @ 0x55dbf3d4da40] error while decoding MB 101 18, bytestream -5 [in#0/rtsp @ 0x55dbf32e0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dbf32e0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:15:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 170.838, "last_started_at": "2026-07-01T21:12:28+09:00", "pid": 23892, "process": {"exists": false, "pid": 23892}, "restart_count": 12, "restart_count_per_hour": 5, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.096, "sample_count": 2135, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 223730, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:15:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T21:15:18+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-01T21:15:18+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-01T21:15:18.742952+09:00", "mtime_epoch": 1782908118.7429523, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:15:18+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 970.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.781, "pushed": 0.781, "read": 0.781}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.438, "count": 14, "max_ms": 6.058, "p95_ms": 6.058}, "gstreamer_push": {"avg_ms": 1.686, "count": 14, "max_ms": 3.746, "p95_ms": 3.746}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.821, "count": 14, "max_ms": 74.88, "p95_ms": 74.88}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.558, "count": 14, "max_ms": 77.437, "p95_ms": 77.437}, "yolo11_640_fire_11class": {"avg_ms": 72.126, "count": 14, "max_ms": 76.738, "p95_ms": 76.738}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.212, "count": 14, "max_ms": 87.764, "p95_ms": 87.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.806, "count": 14, "max_ms": 107.43, "p95_ms": 107.43}, "read_interval": {"avg_ms": 2414.923, "count": 14, "max_ms": 16408.244, "p95_ms": 16408.244}, "triton_wait": {"avg_ms": 81.228, "count": 14, "max_ms": 87.78, "p95_ms": 87.78}, "visualize": {"avg_ms": 5.684, "count": 14, "max_ms": 6.735, "p95_ms": 6.735}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.924, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1864}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4044.461, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:15:18+09:00", "last_error_summary": "[h264 @ 0x55dbf3d4da40] error while decoding MB 101 18, bytestream -5 [in#0/rtsp @ 0x55dbf32e0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dbf32e0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:15:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 170.848, "last_started_at": "2026-07-01T21:12:28+09:00", "pid": 23892, "process": {"exists": false, "pid": 23892}, "restart_count": 12, "restart_count_per_hour": 5, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.096, "sample_count": 2135, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 223730, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:15:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-01T21:15:18+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-01T21:15:18+09:00", "mjpeg": {"age_sec": 0.124, "exists": true, "mtime": "2026-07-01T21:15:18.742952+09:00", "mtime_epoch": 1782908118.7429523, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97377}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:15:18+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 970.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.781, "pushed": 0.781, "read": 0.781}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.438, "count": 14, "max_ms": 6.058, "p95_ms": 6.058}, "gstreamer_push": {"avg_ms": 1.686, "count": 14, "max_ms": 3.746, "p95_ms": 3.746}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.821, "count": 14, "max_ms": 74.88, "p95_ms": 74.88}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.558, "count": 14, "max_ms": 77.437, "p95_ms": 77.437}, "yolo11_640_fire_11class": {"avg_ms": 72.126, "count": 14, "max_ms": 76.738, "p95_ms": 76.738}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.212, "count": 14, "max_ms": 87.764, "p95_ms": 87.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.806, "count": 14, "max_ms": 107.43, "p95_ms": 107.43}, "read_interval": {"avg_ms": 2414.923, "count": 14, "max_ms": 16408.244, "p95_ms": 16408.244}, "triton_wait": {"avg_ms": 81.228, "count": 14, "max_ms": 87.78, "p95_ms": 87.78}, "visualize": {"avg_ms": 5.684, "count": 14, "max_ms": 6.735, "p95_ms": 6.735}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1864}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4062.531, "limit_mb": 8796093022207.996}, "detail": {"pid": 24044, "restart_count": 13, "start_count": 14}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:15:18+09:00", "last_error_summary": "[h264 @ 0x55dbf3d4da40] error while decoding MB 101 18, bytestream -5 [in#0/rtsp @ 0x55dbf32e0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55dbf32e0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:15:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:15:21+09:00", "pid": 24044, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.012, "name": "ffmpeg", "pid": 24044, "rss_mb": 58.012, "state": "S (sleeping)", "threads": 1}, "restart_count": 13, "restart_count_per_hour": 6, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.165, "sample_count": 2090, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 223730, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:15:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.478, "last_frame_pushed_at": "2026-07-01T21:15:18+09:00", "last_frame_read_age_sec": 4.587, "last_frame_read_at": "2026-07-01T21:15:18+09:00", "mjpeg": {"age_sec": 4.461, "exists": true, "mtime": "2026-07-01T21:15:18.866954+09:00", "mtime_epoch": 1782908118.8669538, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96194}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:15:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 970.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.625, "pushed": 0.625, "read": 0.625}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.438, "count": 14, "max_ms": 6.058, "p95_ms": 6.058}, "gstreamer_push": {"avg_ms": 1.686, "count": 14, "max_ms": 3.746, "p95_ms": 3.746}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.821, "count": 14, "max_ms": 74.88, "p95_ms": 74.88}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.558, "count": 14, "max_ms": 77.437, "p95_ms": 77.437}, "yolo11_640_fire_11class": {"avg_ms": 72.126, "count": 14, "max_ms": 76.738, "p95_ms": 76.738}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.212, "count": 14, "max_ms": 87.764, "p95_ms": 87.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.806, "count": 14, "max_ms": 107.43, "p95_ms": 107.43}, "read_interval": {"avg_ms": 2414.923, "count": 14, "max_ms": 16408.244, "p95_ms": 16408.244}, "triton_wait": {"avg_ms": 81.228, "count": 14, "max_ms": 87.78, "p95_ms": 87.78}, "visualize": {"avg_ms": 5.684, "count": 14, "max_ms": 6.735, "p95_ms": 6.735}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.395, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1878}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4066.73, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55de6c882200] missing picture in access unit with size 95 [h264 @ 0x55de6ca0e880] no frame! [in#0/rtsp @ 0x55de6c87de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55de6c87de00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55de6c882840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:18:57+09:00", "last_error_summary": "[NULL @ 0x55de6c882200] missing picture in access unit with size 95 [h264 @ 0x55de6ca0e880] no frame! [in#0/rtsp @ 0x55de6c87de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55de6c87de00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55de6c882840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:15:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 216.503, "last_started_at": "2026-07-01T21:15:21+09:00", "pid": 24044, "process": {"exists": false, "pid": 24044}, "restart_count": 13, "restart_count_per_hour": 6, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.568, "sample_count": 2145, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:18:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-01T21:18:57+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-01T21:18:57+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T21:18:57.766091+09:00", "mtime_epoch": 1782908337.766091, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93756}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:18:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 963.707, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.472, "pushed": 0.472, "read": 0.508}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.625, "count": 14, "max_ms": 3.736, "p95_ms": 3.736}, "gstreamer_push": {"avg_ms": 1.695, "count": 13, "max_ms": 3.724, "p95_ms": 3.724}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.856, "count": 14, "max_ms": 55.528, "p95_ms": 55.528}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.521, "count": 14, "max_ms": 74.237, "p95_ms": 74.237}, "yolo11_640_fire_11class": {"avg_ms": 51.172, "count": 14, "max_ms": 59.05, "p95_ms": 59.05}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.89, "count": 14, "max_ms": 75.542, "p95_ms": 75.542}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.225, "count": 13, "max_ms": 89.626, "p95_ms": 89.626}, "read_interval": {"avg_ms": 2244.412, "count": 14, "max_ms": 16464.433, "p95_ms": 16464.433}, "triton_wait": {"avg_ms": 55.089, "count": 14, "max_ms": 75.555, "p95_ms": 75.555}, "visualize": {"avg_ms": 3.719, "count": 14, "max_ms": 4.275, "p95_ms": 4.275}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.547, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1878}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4066.609, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:18:57+09:00", "last_error_summary": "[NULL @ 0x55de6c882200] missing picture in access unit with size 95 [h264 @ 0x55de6ca0e880] no frame! [in#0/rtsp @ 0x55de6c87de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55de6c87de00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55de6c882840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:18:57+09:00", "last_error_summary": "[NULL @ 0x55de6c882200] missing picture in access unit with size 95 [h264 @ 0x55de6ca0e880] no frame! [in#0/rtsp @ 0x55de6c87de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55de6c87de00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55de6c882840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:18:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 216.52, "last_started_at": "2026-07-01T21:15:21+09:00", "pid": 24044, "process": {"exists": false, "pid": 24044}, "restart_count": 13, "restart_count_per_hour": 6, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.568, "sample_count": 2146, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:18:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-01T21:18:57+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-01T21:18:57+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-01T21:18:57.766091+09:00", "mtime_epoch": 1782908337.766091, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93756}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:18:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 966.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.508, "pushed": 0.508, "read": 0.508}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.625, "count": 14, "max_ms": 3.736, "p95_ms": 3.736}, "gstreamer_push": {"avg_ms": 1.853, "count": 14, "max_ms": 3.897, "p95_ms": 3.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.856, "count": 14, "max_ms": 55.528, "p95_ms": 55.528}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.521, "count": 14, "max_ms": 74.237, "p95_ms": 74.237}, "yolo11_640_fire_11class": {"avg_ms": 51.172, "count": 14, "max_ms": 59.05, "p95_ms": 59.05}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.89, "count": 14, "max_ms": 75.542, "p95_ms": 75.542}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.32, "count": 14, "max_ms": 89.626, "p95_ms": 89.626}, "read_interval": {"avg_ms": 2244.412, "count": 14, "max_ms": 16464.433, "p95_ms": 16464.433}, "triton_wait": {"avg_ms": 55.089, "count": 14, "max_ms": 75.555, "p95_ms": 75.555}, "visualize": {"avg_ms": 3.719, "count": 14, "max_ms": 4.275, "p95_ms": 4.275}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.564, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1878}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4066.355, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:18:57+09:00", "last_error_summary": "[NULL @ 0x55de6c882200] missing picture in access unit with size 95 [h264 @ 0x55de6ca0e880] no frame! [in#0/rtsp @ 0x55de6c87de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55de6c87de00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55de6c882840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:18:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 216.528, "last_started_at": "2026-07-01T21:15:21+09:00", "pid": 24044, "process": {"exists": false, "pid": 24044}, "restart_count": 13, "restart_count_per_hour": 6, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.568, "sample_count": 2146, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:18:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-01T21:18:57+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-01T21:18:57+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-01T21:18:57.766091+09:00", "mtime_epoch": 1782908337.766091, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93756}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:18:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 966.285, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.508, "pushed": 0.508, "read": 0.508}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.625, "count": 14, "max_ms": 3.736, "p95_ms": 3.736}, "gstreamer_push": {"avg_ms": 1.853, "count": 14, "max_ms": 3.897, "p95_ms": 3.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.856, "count": 14, "max_ms": 55.528, "p95_ms": 55.528}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.521, "count": 14, "max_ms": 74.237, "p95_ms": 74.237}, "yolo11_640_fire_11class": {"avg_ms": 51.172, "count": 14, "max_ms": 59.05, "p95_ms": 59.05}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.89, "count": 14, "max_ms": 75.542, "p95_ms": 75.542}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.32, "count": 14, "max_ms": 89.626, "p95_ms": 89.626}, "read_interval": {"avg_ms": 2244.412, "count": 14, "max_ms": 16464.433, "p95_ms": 16464.433}, "triton_wait": {"avg_ms": 55.089, "count": 14, "max_ms": 75.555, "p95_ms": 75.555}, "visualize": {"avg_ms": 3.719, "count": 14, "max_ms": 4.275, "p95_ms": 4.275}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.572, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1878}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4229.953, "limit_mb": 8796093022207.996}, "detail": {"pid": 24196, "restart_count": 14, "start_count": 15}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:18:57+09:00", "last_error_summary": "[NULL @ 0x55de6c882200] missing picture in access unit with size 95 [h264 @ 0x55de6ca0e880] no frame! [in#0/rtsp @ 0x55de6c87de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55de6c87de00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55de6c882840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:18:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:18:59+09:00", "pid": 24196, "process": {"exists": true, "fd_count": 5, "hwm_mb": 201.516, "name": "ffmpeg", "pid": 24196, "rss_mb": 201.516, "state": "S (sleeping)", "threads": 106}, "restart_count": 14, "restart_count_per_hour": 7, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.661, "sample_count": 2109, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:18:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.112, "last_frame_pushed_at": "2026-07-01T21:18:57+09:00", "last_frame_read_age_sec": 4.186, "last_frame_read_at": "2026-07-01T21:18:57+09:00", "mjpeg": {"age_sec": 4.095, "exists": true, "mtime": "2026-07-01T21:18:57.854093+09:00", "mtime_epoch": 1782908337.8540926, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96689}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:19:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 966.285, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.603, "pushed": 0.603, "read": 0.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.619, "count": 13, "max_ms": 3.736, "p95_ms": 3.736}, "gstreamer_push": {"avg_ms": 1.879, "count": 13, "max_ms": 3.897, "p95_ms": 3.897}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.321, "count": 13, "max_ms": 55.528, "p95_ms": 55.528}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.914, "count": 13, "max_ms": 74.237, "p95_ms": 74.237}, "yolo11_640_fire_11class": {"avg_ms": 51.586, "count": 13, "max_ms": 59.05, "p95_ms": 59.05}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.517, "count": 13, "max_ms": 75.542, "p95_ms": 75.542}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.831, "count": 13, "max_ms": 89.626, "p95_ms": 89.626}, "read_interval": {"avg_ms": 2113.792, "count": 13, "max_ms": 16464.433, "p95_ms": 16464.433}, "triton_wait": {"avg_ms": 55.602, "count": 13, "max_ms": 75.555, "p95_ms": 75.555}, "visualize": {"avg_ms": 3.719, "count": 13, "max_ms": 4.275, "p95_ms": 4.275}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.547, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 907}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4091.711, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55af44704380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af44704380] error while decoding MB 0 18, bytestream 3090 [h264 @ 0x55af43d06240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af43d06240] error while decoding MB 0 40, bytestream 1355 [h264 @ 0x55af43cc3480] error while decoding MB 75 36, bytestream -10 [in#0/rtsp @ 0x55af43c7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af43c7fe00] Error retrieving a packet from d", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 158, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:19:54+09:00", "last_error_summary": "[h264 @ 0x55af44704380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af44704380] error while decoding MB 0 18, bytestream 3090 [h264 @ 0x55af43d06240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af43d06240] error while decoding MB 0 40, bytestream 1355 [h264 @ 0x55af43cc3480] error while decoding MB 75 36, bytestream -10 [in#0/rtsp @ 0x55af43c7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af43c7fe00] Error retrieving a packet from d", "last_exit_at": "2026-07-01T21:08:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 673.993, "last_started_at": "2026-07-01T21:08:40+09:00", "pid": 23633, "process": {"exists": false, "pid": 23633}, "restart_count": 182, "restart_count_per_hour": 12, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 183}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.416, "sample_count": 2831, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108890, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:19:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-01T21:19:54+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-01T21:19:54+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-01T21:19:54.310962+09:00", "mtime_epoch": 1782908394.3109622, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 110567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:19:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 914.996, "name": "celery", "pid": 237, "rss_mb": 910.043, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.407, "pushed": 3.407, "read": 3.407}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.558, "count": 102, "max_ms": 2.02, "p95_ms": 2.001}, "gstreamer_push": {"avg_ms": 1.679, "count": 102, "max_ms": 6.28, "p95_ms": 3.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.024, "count": 102, "max_ms": 78.268, "p95_ms": 76.408}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.165, "count": 102, "max_ms": 81.788, "p95_ms": 78.754}, "yolo11_640_fire_11class": {"avg_ms": 72.907, "count": 102, "max_ms": 82.344, "p95_ms": 79.457}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.267, "count": 102, "max_ms": 90.934, "p95_ms": 87.737}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.213, "count": 102, "max_ms": 102.971, "p95_ms": 99.159}, "read_interval": {"avg_ms": 293.866, "count": 102, "max_ms": 10125.579, "p95_ms": 119.097}, "triton_wait": {"avg_ms": 80.452, "count": 102, "max_ms": 90.948, "p95_ms": 87.749}, "visualize": {"avg_ms": 3.737, "count": 102, "max_ms": 4.214, "p95_ms": 4.045}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 907}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4091.543, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:19:54+09:00", "last_error_summary": "[h264 @ 0x55af44704380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af44704380] error while decoding MB 0 18, bytestream 3090 [h264 @ 0x55af43d06240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af43d06240] error while decoding MB 0 40, bytestream 1355 [h264 @ 0x55af43cc3480] error while decoding MB 75 36, bytestream -10 [in#0/rtsp @ 0x55af43c7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af43c7fe00] Error retrieving a packet from d", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 158, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:19:54+09:00", "last_error_summary": "[h264 @ 0x55af44704380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af44704380] error while decoding MB 0 18, bytestream 3090 [h264 @ 0x55af43d06240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af43d06240] error while decoding MB 0 40, bytestream 1355 [h264 @ 0x55af43cc3480] error while decoding MB 75 36, bytestream -10 [in#0/rtsp @ 0x55af43c7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af43c7fe00] Error retrieving a packet from d", "last_exit_at": "2026-07-01T21:19:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 674.022, "last_started_at": "2026-07-01T21:08:40+09:00", "pid": 23633, "process": {"exists": false, "pid": 23633}, "restart_count": 182, "restart_count_per_hour": 12, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 183}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.416, "sample_count": 2832, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108891, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:19:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T21:19:54+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-01T21:19:54+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-01T21:19:54.310962+09:00", "mtime_epoch": 1782908394.3109622, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 110567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:19:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 914.996, "name": "celery", "pid": 237, "rss_mb": 910.043, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.437, "pushed": 3.437, "read": 3.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.562, "count": 103, "max_ms": 2.02, "p95_ms": 2.001}, "gstreamer_push": {"avg_ms": 1.676, "count": 103, "max_ms": 6.28, "p95_ms": 3.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.072, "count": 103, "max_ms": 78.268, "p95_ms": 76.408}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.096, "count": 103, "max_ms": 81.788, "p95_ms": 78.754}, "yolo11_640_fire_11class": {"avg_ms": 72.772, "count": 103, "max_ms": 82.344, "p95_ms": 79.457}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.128, "count": 103, "max_ms": 90.934, "p95_ms": 87.737}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.094, "count": 103, "max_ms": 102.971, "p95_ms": 99.159}, "read_interval": {"avg_ms": 293.866, "count": 102, "max_ms": 10125.579, "p95_ms": 119.097}, "triton_wait": {"avg_ms": 80.312, "count": 103, "max_ms": 90.948, "p95_ms": 87.749}, "visualize": {"avg_ms": 3.747, "count": 103, "max_ms": 4.825, "p95_ms": 4.046}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 907}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4091.535, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 158, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:19:54+09:00", "last_error_summary": "[h264 @ 0x55af44704380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af44704380] error while decoding MB 0 18, bytestream 3090 [h264 @ 0x55af43d06240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af43d06240] error while decoding MB 0 40, bytestream 1355 [h264 @ 0x55af43cc3480] error while decoding MB 75 36, bytestream -10 [in#0/rtsp @ 0x55af43c7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af43c7fe00] Error retrieving a packet from d", "last_exit_at": "2026-07-01T21:19:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 674.03, "last_started_at": "2026-07-01T21:08:40+09:00", "pid": 23633, "process": {"exists": false, "pid": 23633}, "restart_count": 182, "restart_count_per_hour": 12, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 183}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.416, "sample_count": 2832, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108891, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:19:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T21:19:54+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-01T21:19:54+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T21:19:54.310962+09:00", "mtime_epoch": 1782908394.3109622, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 110567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:19:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 914.996, "name": "celery", "pid": 237, "rss_mb": 910.043, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.436, "pushed": 3.436, "read": 3.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.562, "count": 103, "max_ms": 2.02, "p95_ms": 2.001}, "gstreamer_push": {"avg_ms": 1.676, "count": 103, "max_ms": 6.28, "p95_ms": 3.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.072, "count": 103, "max_ms": 78.268, "p95_ms": 76.408}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.096, "count": 103, "max_ms": 81.788, "p95_ms": 78.754}, "yolo11_640_fire_11class": {"avg_ms": 72.772, "count": 103, "max_ms": 82.344, "p95_ms": 79.457}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.128, "count": 103, "max_ms": 90.934, "p95_ms": 87.737}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.094, "count": 103, "max_ms": 102.971, "p95_ms": 99.159}, "read_interval": {"avg_ms": 293.866, "count": 102, "max_ms": 10125.579, "p95_ms": 119.097}, "triton_wait": {"avg_ms": 80.312, "count": 103, "max_ms": 90.948, "p95_ms": 87.749}, "visualize": {"avg_ms": 3.747, "count": 103, "max_ms": 4.825, "p95_ms": 4.046}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 907}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4198.996, "limit_mb": 8796093022207.996}, "detail": {"pid": 24348, "restart_count": 183, "start_count": 184}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 158, "h264_decode_error_count": 175, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:19:54+09:00", "last_error_summary": "[h264 @ 0x55af44704380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af44704380] error while decoding MB 0 18, bytestream 3090 [h264 @ 0x55af43d06240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af43d06240] error while decoding MB 0 40, bytestream 1355 [h264 @ 0x55af43cc3480] error while decoding MB 75 36, bytestream -10 [in#0/rtsp @ 0x55af43c7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55af43c7fe00] Error retrieving a packet from d", "last_exit_at": "2026-07-01T21:19:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:19:56+09:00", "pid": 24348, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.664, "name": "ffmpeg", "pid": 24348, "rss_mb": 138.664, "state": "S (sleeping)", "threads": 106}, "restart_count": 183, "restart_count_per_hour": 13, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 184}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.463, "sample_count": 2772, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108891, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:19:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.527, "last_frame_pushed_at": "2026-07-01T21:19:54+09:00", "last_frame_read_age_sec": 4.608, "last_frame_read_at": "2026-07-01T21:19:54+09:00", "mjpeg": {"age_sec": 4.511, "exists": true, "mtime": "2026-07-01T21:19:54.402964+09:00", "mtime_epoch": 1782908394.4029636, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 109913}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:19:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 914.996, "name": "celery", "pid": 237, "rss_mb": 910.043, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.073, "pushed": 2.073, "read": 2.039}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.541, "count": 62, "max_ms": 1.952, "p95_ms": 1.652}, "gstreamer_push": {"avg_ms": 1.674, "count": 62, "max_ms": 6.28, "p95_ms": 3.834}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.021, "count": 62, "max_ms": 76.408, "p95_ms": 74.172}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.189, "count": 62, "max_ms": 80.328, "p95_ms": 78.519}, "yolo11_640_fire_11class": {"avg_ms": 70.788, "count": 62, "max_ms": 81.084, "p95_ms": 78.299}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.918, "count": 62, "max_ms": 90.934, "p95_ms": 88.244}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.001, "count": 62, "max_ms": 102.971, "p95_ms": 99.193}, "read_interval": {"avg_ms": 416.37, "count": 61, "max_ms": 10125.579, "p95_ms": 119.097}, "triton_wait": {"avg_ms": 78.214, "count": 62, "max_ms": 90.948, "p95_ms": 88.257}, "visualize": {"avg_ms": 3.738, "count": 62, "max_ms": 4.825, "p95_ms": 4.045}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1890}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4338.16, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:18:57+09:00", "last_error_summary": "[NULL @ 0x55de6c882200] missing picture in access unit with size 95 [h264 @ 0x55de6ca0e880] no frame! [in#0/rtsp @ 0x55de6c87de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55de6c87de00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55de6c882840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:18:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 181.339, "last_started_at": "2026-07-01T21:18:59+09:00", "pid": 24196, "process": {"exists": true, "fd_count": 5, "hwm_mb": 325.32, "name": "ffmpeg", "pid": 24196, "rss_mb": 325.32, "state": "S (sleeping)", "threads": 106}, "restart_count": 14, "restart_count_per_hour": 7, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.915, "sample_count": 2201, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226855, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:21:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.058, "last_frame_pushed_at": "2026-07-01T21:21:31+09:00", "last_frame_read_age_sec": 30.13, "last_frame_read_at": "2026-07-01T21:21:31+09:00", "mjpeg": {"age_sec": 30.043, "exists": true, "mtime": "2026-07-01T21:21:31.244500+09:00", "mtime_epoch": 1782908491.2445004, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99395}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:22:01+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 972.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1890}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4326.367, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:18:57+09:00", "last_error_summary": "[NULL @ 0x55de6c882200] missing picture in access unit with size 95 [h264 @ 0x55de6ca0e880] no frame! [in#0/rtsp @ 0x55de6c87de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55de6c87de00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55de6c882840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:18:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 181.352, "last_started_at": "2026-07-01T21:18:59+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 14, "restart_count_per_hour": 7, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.915, "sample_count": 2201, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226855, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:21:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.07, "last_frame_pushed_at": "2026-07-01T21:21:31+09:00", "last_frame_read_age_sec": 30.142, "last_frame_read_at": "2026-07-01T21:21:31+09:00", "mjpeg": {"age_sec": 30.055, "exists": true, "mtime": "2026-07-01T21:21:31.244500+09:00", "mtime_epoch": 1782908491.2445004, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99395}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:22:01+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 972.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1890}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4040.723, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5600c2906000] error while decoding MB 113 45, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:22:01+09:00", "last_error_summary": "[h264 @ 0x5600c2906000] error while decoding MB 113 45, bytestream -9", "last_exit_at": "2026-07-01T21:18:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 181.403, "last_started_at": "2026-07-01T21:18:59+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 14, "restart_count_per_hour": 7, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.913, "sample_count": 2200, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226855, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:21:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.121, "last_frame_pushed_at": "2026-07-01T21:21:31+09:00", "last_frame_read_age_sec": 30.193, "last_frame_read_at": "2026-07-01T21:21:31+09:00", "mjpeg": {"age_sec": 30.106, "exists": true, "mtime": "2026-07-01T21:21:31.244500+09:00", "mtime_epoch": 1782908491.2445004, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99395}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:22:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 972.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1890}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4060.191, "limit_mb": 8796093022207.996}, "detail": {"pid": 24455, "restart_count": 15, "start_count": 16}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:22:01+09:00", "last_error_summary": "[h264 @ 0x5600c2906000] error while decoding MB 113 45, bytestream -9", "last_exit_at": "2026-07-01T21:18:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:22:03+09:00", "pid": 24455, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.398, "name": "ffmpeg", "pid": 24455, "rss_mb": 57.398, "state": "S (sleeping)", "threads": 1}, "restart_count": 15, "restart_count_per_hour": 8, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.84, "sample_count": 2164, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226855, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:21:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 34.222, "last_frame_pushed_at": "2026-07-01T21:21:31+09:00", "last_frame_read_age_sec": 34.294, "last_frame_read_at": "2026-07-01T21:21:31+09:00", "mjpeg": {"age_sec": 34.207, "exists": true, "mtime": "2026-07-01T21:21:31.244500+09:00", "mtime_epoch": 1782908491.2445004, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99395}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:22:05+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 972.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1901}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4049.109, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55bb61813240] error while decoding MB 45 67, bytestream -5 [in#0/rtsp @ 0x55bb60f4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb60f4ae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:24:57+09:00", "last_error_summary": "[h264 @ 0x55bb61813240] error while decoding MB 45 67, bytestream -5 [in#0/rtsp @ 0x55bb60f4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb60f4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:18:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 174.318, "last_started_at": "2026-07-01T21:22:03+09:00", "pid": 24455, "process": {"exists": false, "pid": 24455}, "restart_count": 15, "restart_count_per_hour": 8, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.067, "sample_count": 2124, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228129, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:24:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T21:24:57+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-01T21:24:57+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-01T21:24:57.711928+09:00", "mtime_epoch": 1782908697.7119277, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94582}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:24:57+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 972.805, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.753, "pushed": 0.753, "read": 0.811}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.951, "count": 13, "max_ms": 5.936, "p95_ms": 5.936}, "gstreamer_push": {"avg_ms": 1.312, "count": 13, "max_ms": 1.385, "p95_ms": 1.385}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.696, "count": 13, "max_ms": 57.346, "p95_ms": 57.346}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.659, "count": 13, "max_ms": 74.015, "p95_ms": 74.015}, "yolo11_640_fire_11class": {"avg_ms": 56.976, "count": 13, "max_ms": 75.786, "p95_ms": 75.786}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.159, "count": 13, "max_ms": 81.247, "p95_ms": 81.247}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.035, "count": 13, "max_ms": 96.282, "p95_ms": 96.282}, "read_interval": {"avg_ms": 2447.677, "count": 14, "max_ms": 17061.355, "p95_ms": 17061.355}, "triton_wait": {"avg_ms": 59.847, "count": 13, "max_ms": 81.26, "p95_ms": 81.26}, "visualize": {"avg_ms": 3.976, "count": 13, "max_ms": 6.422, "p95_ms": 6.422}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.27, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1901}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4048.57, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:24:57+09:00", "last_error_summary": "[h264 @ 0x55bb61813240] error while decoding MB 45 67, bytestream -5 [in#0/rtsp @ 0x55bb60f4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb60f4ae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:24:57+09:00", "last_error_summary": "[h264 @ 0x55bb61813240] error while decoding MB 45 67, bytestream -5 [in#0/rtsp @ 0x55bb60f4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb60f4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:24:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 174.365, "last_started_at": "2026-07-01T21:22:03+09:00", "pid": 24455, "process": {"exists": false, "pid": 24455}, "restart_count": 15, "restart_count_per_hour": 8, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.067, "sample_count": 2125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228130, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:24:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T21:24:57+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T21:24:57+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-01T21:24:57.711928+09:00", "mtime_epoch": 1782908697.7119277, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94582}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:24:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 972.805, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.809, "pushed": 0.809, "read": 0.809}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.919, "count": 14, "max_ms": 5.936, "p95_ms": 5.936}, "gstreamer_push": {"avg_ms": 1.311, "count": 14, "max_ms": 1.385, "p95_ms": 1.385}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.776, "count": 14, "max_ms": 76.807, "p95_ms": 76.807}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.036, "count": 14, "max_ms": 77.929, "p95_ms": 77.929}, "yolo11_640_fire_11class": {"avg_ms": 58.583, "count": 14, "max_ms": 79.463, "p95_ms": 79.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.143, "count": 14, "max_ms": 86.927, "p95_ms": 86.927}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.917, "count": 14, "max_ms": 99.391, "p95_ms": 99.391}, "read_interval": {"avg_ms": 2447.677, "count": 14, "max_ms": 17061.355, "p95_ms": 17061.355}, "triton_wait": {"avg_ms": 61.782, "count": 14, "max_ms": 86.939, "p95_ms": 86.939}, "visualize": {"avg_ms": 3.949, "count": 14, "max_ms": 6.422, "p95_ms": 6.422}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.316, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1901}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4048.434, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:24:57+09:00", "last_error_summary": "[h264 @ 0x55bb61813240] error while decoding MB 45 67, bytestream -5 [in#0/rtsp @ 0x55bb60f4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb60f4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:24:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 174.371, "last_started_at": "2026-07-01T21:22:03+09:00", "pid": 24455, "process": {"exists": false, "pid": 24455}, "restart_count": 15, "restart_count_per_hour": 8, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.067, "sample_count": 2125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228130, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:24:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T21:24:57+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-01T21:24:57+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T21:24:57.711928+09:00", "mtime_epoch": 1782908697.7119277, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94582}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:24:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 972.805, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.808, "pushed": 0.808, "read": 0.808}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.919, "count": 14, "max_ms": 5.936, "p95_ms": 5.936}, "gstreamer_push": {"avg_ms": 1.311, "count": 14, "max_ms": 1.385, "p95_ms": 1.385}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.776, "count": 14, "max_ms": 76.807, "p95_ms": 76.807}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.036, "count": 14, "max_ms": 77.929, "p95_ms": 77.929}, "yolo11_640_fire_11class": {"avg_ms": 58.583, "count": 14, "max_ms": 79.463, "p95_ms": 79.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.143, "count": 14, "max_ms": 86.927, "p95_ms": 86.927}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.917, "count": 14, "max_ms": 99.391, "p95_ms": 99.391}, "read_interval": {"avg_ms": 2447.677, "count": 14, "max_ms": 17061.355, "p95_ms": 17061.355}, "triton_wait": {"avg_ms": 61.782, "count": 14, "max_ms": 86.939, "p95_ms": 86.939}, "visualize": {"avg_ms": 3.949, "count": 14, "max_ms": 6.422, "p95_ms": 6.422}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.322, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1901}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4126.039, "limit_mb": 8796093022207.996}, "detail": {"pid": 24607, "restart_count": 16, "start_count": 17}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:24:57+09:00", "last_error_summary": "[h264 @ 0x55bb61813240] error while decoding MB 45 67, bytestream -5 [in#0/rtsp @ 0x55bb60f4ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bb60f4ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:25:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:25:00+09:00", "pid": 24607, "process": {"exists": true, "fd_count": 5, "hwm_mb": 108.223, "name": "ffmpeg", "pid": 24607, "rss_mb": 108.223, "state": "S (sleeping)", "threads": 19}, "restart_count": 16, "restart_count_per_hour": 9, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.092, "sample_count": 2081, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228130, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:24:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.878, "last_frame_pushed_at": "2026-07-01T21:24:57+09:00", "last_frame_read_age_sec": 4.979, "last_frame_read_at": "2026-07-01T21:24:57+09:00", "mjpeg": {"age_sec": 4.857, "exists": true, "mtime": "2026-07-01T21:24:57.827930+09:00", "mtime_epoch": 1782908697.8279297, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95067}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:25:02+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 972.805, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.631, "pushed": 0.631, "read": 0.631}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.919, "count": 14, "max_ms": 5.936, "p95_ms": 5.936}, "gstreamer_push": {"avg_ms": 1.311, "count": 14, "max_ms": 1.385, "p95_ms": 1.385}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.776, "count": 14, "max_ms": 76.807, "p95_ms": 76.807}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.036, "count": 14, "max_ms": 77.929, "p95_ms": 77.929}, "yolo11_640_fire_11class": {"avg_ms": 58.583, "count": 14, "max_ms": 79.463, "p95_ms": 79.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.143, "count": 14, "max_ms": 86.927, "p95_ms": 86.927}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.917, "count": 14, "max_ms": 99.391, "p95_ms": 99.391}, "read_interval": {"avg_ms": 2447.677, "count": 14, "max_ms": 17061.355, "p95_ms": 17061.355}, "triton_wait": {"avg_ms": 61.782, "count": 14, "max_ms": 86.939, "p95_ms": 86.939}, "visualize": {"avg_ms": 3.949, "count": 14, "max_ms": 6.422, "p95_ms": 6.422}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.186, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1907}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4052.836, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x556a2ebfd200] missing picture in access unit with size 95 [h264 @ 0x556a2ed7f940] no frame! [in#0/rtsp @ 0x556a2ebf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556a2ebf8e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x556a2ebfd840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:27:03+09:00", "last_error_summary": "[NULL @ 0x556a2ebfd200] missing picture in access unit with size 95 [h264 @ 0x556a2ed7f940] no frame! [in#0/rtsp @ 0x556a2ebf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556a2ebf8e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x556a2ebfd840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:25:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.952, "last_started_at": "2026-07-01T21:25:00+09:00", "pid": 24607, "process": {"exists": false, "pid": 24607}, "restart_count": 16, "restart_count_per_hour": 9, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.395, "sample_count": 2053, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228908, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:27:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T21:27:03+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-01T21:27:03+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T21:27:03.574094+09:00", "mtime_epoch": 1782908823.5740936, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97445}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:27:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 973.52, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.665, "pushed": 0.665, "read": 0.72}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.601, "count": 12, "max_ms": 3.74, "p95_ms": 3.74}, "gstreamer_push": {"avg_ms": 1.349, "count": 12, "max_ms": 1.442, "p95_ms": 1.442}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.252, "count": 12, "max_ms": 66.38, "p95_ms": 66.38}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.411, "count": 12, "max_ms": 76.949, "p95_ms": 76.949}, "yolo11_640_fire_11class": {"avg_ms": 67.655, "count": 12, "max_ms": 76.894, "p95_ms": 76.894}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.05, "count": 12, "max_ms": 85.051, "p95_ms": 85.051}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.704, "count": 12, "max_ms": 97.482, "p95_ms": 97.482}, "read_interval": {"avg_ms": 2699.106, "count": 13, "max_ms": 17108.158, "p95_ms": 17108.158}, "triton_wait": {"avg_ms": 74.21, "count": 12, "max_ms": 85.064, "p95_ms": 85.064}, "visualize": {"avg_ms": 3.689, "count": 12, "max_ms": 3.817, "p95_ms": 3.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.043, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1907}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4052.426, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:27:03+09:00", "last_error_summary": "[NULL @ 0x556a2ebfd200] missing picture in access unit with size 95 [h264 @ 0x556a2ed7f940] no frame! [in#0/rtsp @ 0x556a2ebf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556a2ebf8e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x556a2ebfd840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:27:03+09:00", "last_error_summary": "[NULL @ 0x556a2ebfd200] missing picture in access unit with size 95 [h264 @ 0x556a2ed7f940] no frame! [in#0/rtsp @ 0x556a2ebf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556a2ebf8e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x556a2ebfd840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:27:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.993, "last_started_at": "2026-07-01T21:25:00+09:00", "pid": 24607, "process": {"exists": false, "pid": 24607}, "restart_count": 16, "restart_count_per_hour": 9, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.395, "sample_count": 2054, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228909, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:27:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T21:27:03+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-01T21:27:03+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-01T21:27:03.574094+09:00", "mtime_epoch": 1782908823.5740936, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97445}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:27:03+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 973.52, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.719, "pushed": 0.719, "read": 0.719}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.59, "count": 13, "max_ms": 3.74, "p95_ms": 3.74}, "gstreamer_push": {"avg_ms": 1.352, "count": 13, "max_ms": 1.442, "p95_ms": 1.442}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.293, "count": 13, "max_ms": 66.38, "p95_ms": 66.38}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.963, "count": 13, "max_ms": 76.949, "p95_ms": 76.949}, "yolo11_640_fire_11class": {"avg_ms": 68.142, "count": 13, "max_ms": 76.894, "p95_ms": 76.894}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.795, "count": 13, "max_ms": 85.051, "p95_ms": 85.051}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.422, "count": 13, "max_ms": 97.482, "p95_ms": 97.482}, "read_interval": {"avg_ms": 2699.106, "count": 13, "max_ms": 17108.158, "p95_ms": 17108.158}, "triton_wait": {"avg_ms": 74.943, "count": 13, "max_ms": 85.064, "p95_ms": 85.064}, "visualize": {"avg_ms": 3.686, "count": 13, "max_ms": 3.817, "p95_ms": 3.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.084, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1907}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4052.559, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:27:03+09:00", "last_error_summary": "[NULL @ 0x556a2ebfd200] missing picture in access unit with size 95 [h264 @ 0x556a2ed7f940] no frame! [in#0/rtsp @ 0x556a2ebf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556a2ebf8e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x556a2ebfd840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:27:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.0, "last_started_at": "2026-07-01T21:25:00+09:00", "pid": 24607, "process": {"exists": false, "pid": 24607}, "restart_count": 16, "restart_count_per_hour": 9, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.395, "sample_count": 2054, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228909, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:27:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T21:27:03+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T21:27:03+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T21:27:03.574094+09:00", "mtime_epoch": 1782908823.5740936, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97445}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:27:03+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 973.52, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.719, "pushed": 0.719, "read": 0.719}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.59, "count": 13, "max_ms": 3.74, "p95_ms": 3.74}, "gstreamer_push": {"avg_ms": 1.352, "count": 13, "max_ms": 1.442, "p95_ms": 1.442}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.293, "count": 13, "max_ms": 66.38, "p95_ms": 66.38}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.963, "count": 13, "max_ms": 76.949, "p95_ms": 76.949}, "yolo11_640_fire_11class": {"avg_ms": 68.142, "count": 13, "max_ms": 76.894, "p95_ms": 76.894}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.795, "count": 13, "max_ms": 85.051, "p95_ms": 85.051}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.422, "count": 13, "max_ms": 97.482, "p95_ms": 97.482}, "read_interval": {"avg_ms": 2699.106, "count": 13, "max_ms": 17108.158, "p95_ms": 17108.158}, "triton_wait": {"avg_ms": 74.943, "count": 13, "max_ms": 85.064, "p95_ms": 85.064}, "visualize": {"avg_ms": 3.686, "count": 13, "max_ms": 3.817, "p95_ms": 3.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.091, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1907}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4081.984, "limit_mb": 8796093022207.996}, "detail": {"pid": 24759, "restart_count": 17, "start_count": 18}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:27:03+09:00", "last_error_summary": "[NULL @ 0x556a2ebfd200] missing picture in access unit with size 95 [h264 @ 0x556a2ed7f940] no frame! [in#0/rtsp @ 0x556a2ebf8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556a2ebf8e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x556a2ebfd840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:27:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:27:06+09:00", "pid": 24759, "process": {"exists": true, "fd_count": 5, "hwm_mb": 66.641, "name": "ffmpeg", "pid": 24759, "rss_mb": 66.641, "state": "S (sleeping)", "threads": 19}, "restart_count": 17, "restart_count_per_hour": 10, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.409, "sample_count": 2029, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228909, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:27:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.914, "last_frame_pushed_at": "2026-07-01T21:27:03+09:00", "last_frame_read_age_sec": 5.012, "last_frame_read_at": "2026-07-01T21:27:03+09:00", "mjpeg": {"age_sec": 4.899, "exists": true, "mtime": "2026-07-01T21:27:03.686095+09:00", "mtime_epoch": 1782908823.6860955, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97974}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:27:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 973.52, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.565, "pushed": 0.565, "read": 0.565}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.59, "count": 13, "max_ms": 3.74, "p95_ms": 3.74}, "gstreamer_push": {"avg_ms": 1.352, "count": 13, "max_ms": 1.442, "p95_ms": 1.442}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.293, "count": 13, "max_ms": 66.38, "p95_ms": 66.38}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.963, "count": 13, "max_ms": 76.949, "p95_ms": 76.949}, "yolo11_640_fire_11class": {"avg_ms": 68.142, "count": 13, "max_ms": 76.894, "p95_ms": 76.894}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.795, "count": 13, "max_ms": 85.051, "p95_ms": 85.051}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.422, "count": 13, "max_ms": 97.482, "p95_ms": 97.482}, "read_interval": {"avg_ms": 2699.106, "count": 13, "max_ms": 17108.158, "p95_ms": 17108.158}, "triton_wait": {"avg_ms": 74.943, "count": 13, "max_ms": 85.064, "p95_ms": 85.064}, "visualize": {"avg_ms": 3.686, "count": 13, "max_ms": 3.817, "p95_ms": 3.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4051.379, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5629b7ca4480] error while decoding MB 87 19, bytestream -5 [in#0/rtsp @ 0x5629b73d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629b73d6e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:29:21+09:00", "last_error_summary": "[h264 @ 0x5629b7ca4480] error while decoding MB 87 19, bytestream -5 [in#0/rtsp @ 0x5629b73d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629b73d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:27:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.755, "last_started_at": "2026-07-01T21:27:06+09:00", "pid": 24759, "process": {"exists": false, "pid": 24759}, "restart_count": 17, "restart_count_per_hour": 10, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.079, "sample_count": 1713, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 229809, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:29:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-01T21:29:21+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-01T21:29:21+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-01T21:29:21.284512+09:00", "mtime_epoch": 1782908961.2845118, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90934}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:29:21+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 973.719, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.688, "pushed": 0.688, "read": 0.746}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.666, "count": 12, "max_ms": 3.986, "p95_ms": 3.986}, "gstreamer_push": {"avg_ms": 1.573, "count": 12, "max_ms": 3.9, "p95_ms": 3.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.099, "count": 12, "max_ms": 79.86, "p95_ms": 79.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.38, "count": 12, "max_ms": 82.826, "p95_ms": 82.826}, "yolo11_640_fire_11class": {"avg_ms": 70.65, "count": 12, "max_ms": 76.282, "p95_ms": 76.282}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.301, "count": 12, "max_ms": 86.029, "p95_ms": 86.029}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.55, "count": 12, "max_ms": 100.425, "p95_ms": 100.425}, "read_interval": {"avg_ms": 2624.157, "count": 13, "max_ms": 16732.083, "p95_ms": 16732.083}, "triton_wait": {"avg_ms": 78.446, "count": 12, "max_ms": 86.042, "p95_ms": 86.042}, "visualize": {"avg_ms": 3.725, "count": 12, "max_ms": 3.947, "p95_ms": 3.947}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.438, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4050.879, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:29:21+09:00", "last_error_summary": "[h264 @ 0x5629b7ca4480] error while decoding MB 87 19, bytestream -5 [in#0/rtsp @ 0x5629b73d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629b73d6e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:29:21+09:00", "last_error_summary": "[h264 @ 0x5629b7ca4480] error while decoding MB 87 19, bytestream -5 [in#0/rtsp @ 0x5629b73d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629b73d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:29:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.805, "last_started_at": "2026-07-01T21:27:06+09:00", "pid": 24759, "process": {"exists": false, "pid": 24759}, "restart_count": 17, "restart_count_per_hour": 10, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.079, "sample_count": 1714, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 229810, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:29:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T21:29:21+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-01T21:29:21+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-01T21:29:21.284512+09:00", "mtime_epoch": 1782908961.2845118, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90934}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:29:21+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 973.719, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.743, "pushed": 0.743, "read": 0.743}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.663, "count": 13, "max_ms": 3.986, "p95_ms": 3.986}, "gstreamer_push": {"avg_ms": 1.561, "count": 13, "max_ms": 3.9, "p95_ms": 3.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.429, "count": 13, "max_ms": 79.86, "p95_ms": 79.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.595, "count": 13, "max_ms": 82.826, "p95_ms": 82.826}, "yolo11_640_fire_11class": {"avg_ms": 71.055, "count": 13, "max_ms": 76.282, "p95_ms": 76.282}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.795, "count": 13, "max_ms": 86.029, "p95_ms": 86.029}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.046, "count": 13, "max_ms": 100.425, "p95_ms": 100.425}, "read_interval": {"avg_ms": 2624.157, "count": 13, "max_ms": 16732.083, "p95_ms": 16732.083}, "triton_wait": {"avg_ms": 78.929, "count": 13, "max_ms": 86.042, "p95_ms": 86.042}, "visualize": {"avg_ms": 3.72, "count": 13, "max_ms": 3.947, "p95_ms": 3.947}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.488, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4050.902, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:29:21+09:00", "last_error_summary": "[h264 @ 0x5629b7ca4480] error while decoding MB 87 19, bytestream -5 [in#0/rtsp @ 0x5629b73d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629b73d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:29:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.812, "last_started_at": "2026-07-01T21:27:06+09:00", "pid": 24759, "process": {"exists": false, "pid": 24759}, "restart_count": 17, "restart_count_per_hour": 10, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.079, "sample_count": 1714, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 229810, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:29:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T21:29:21+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T21:29:21+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-01T21:29:21.284512+09:00", "mtime_epoch": 1782908961.2845118, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90934}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:29:21+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 973.719, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.743, "pushed": 0.743, "read": 0.743}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.663, "count": 13, "max_ms": 3.986, "p95_ms": 3.986}, "gstreamer_push": {"avg_ms": 1.561, "count": 13, "max_ms": 3.9, "p95_ms": 3.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.429, "count": 13, "max_ms": 79.86, "p95_ms": 79.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.595, "count": 13, "max_ms": 82.826, "p95_ms": 82.826}, "yolo11_640_fire_11class": {"avg_ms": 71.055, "count": 13, "max_ms": 76.282, "p95_ms": 76.282}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.795, "count": 13, "max_ms": 86.029, "p95_ms": 86.029}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.046, "count": 13, "max_ms": 100.425, "p95_ms": 100.425}, "read_interval": {"avg_ms": 2624.157, "count": 13, "max_ms": 16732.083, "p95_ms": 16732.083}, "triton_wait": {"avg_ms": 78.929, "count": 13, "max_ms": 86.042, "p95_ms": 86.042}, "visualize": {"avg_ms": 3.72, "count": 13, "max_ms": 3.947, "p95_ms": 3.947}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.495, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1915}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4223.434, "limit_mb": 8796093022207.996}, "detail": {"pid": 24911, "restart_count": 18, "start_count": 19}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:29:21+09:00", "last_error_summary": "[h264 @ 0x5629b7ca4480] error while decoding MB 87 19, bytestream -5 [in#0/rtsp @ 0x5629b73d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629b73d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:29:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:29:23+09:00", "pid": 24911, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.43, "name": "ffmpeg", "pid": 24911, "rss_mb": 197.43, "state": "S (sleeping)", "threads": 106}, "restart_count": 18, "restart_count_per_hour": 11, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.079, "sample_count": 1694, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 229810, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:29:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.507, "last_frame_pushed_at": "2026-07-01T21:29:21+09:00", "last_frame_read_age_sec": 4.609, "last_frame_read_at": "2026-07-01T21:29:21+09:00", "mjpeg": {"age_sec": 4.492, "exists": true, "mtime": "2026-07-01T21:29:21.400514+09:00", "mtime_epoch": 1782908961.400514, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92532}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:29:25+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 973.719, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.591, "pushed": 0.591, "read": 0.591}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.663, "count": 13, "max_ms": 3.986, "p95_ms": 3.986}, "gstreamer_push": {"avg_ms": 1.561, "count": 13, "max_ms": 3.9, "p95_ms": 3.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.429, "count": 13, "max_ms": 79.86, "p95_ms": 79.86}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.595, "count": 13, "max_ms": 82.826, "p95_ms": 82.826}, "yolo11_640_fire_11class": {"avg_ms": 71.055, "count": 13, "max_ms": 76.282, "p95_ms": 76.282}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.795, "count": 13, "max_ms": 86.029, "p95_ms": 86.029}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.046, "count": 13, "max_ms": 100.425, "p95_ms": 100.425}, "read_interval": {"avg_ms": 2624.157, "count": 13, "max_ms": 16732.083, "p95_ms": 16732.083}, "triton_wait": {"avg_ms": 78.929, "count": 13, "max_ms": 86.042, "p95_ms": 86.042}, "visualize": {"avg_ms": 3.72, "count": 13, "max_ms": 3.947, "p95_ms": 3.947}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1922}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4064.234, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f464b68900] error while decoding MB 0 57, bytestream -11 [in#0/rtsp @ 0x55f4645d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f4645d6e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:31:39+09:00", "last_error_summary": "[h264 @ 0x55f464b68900] error while decoding MB 0 57, bytestream -11 [in#0/rtsp @ 0x55f4645d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f4645d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:29:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.067, "last_started_at": "2026-07-01T21:29:23+09:00", "pid": 24911, "process": {"exists": false, "pid": 24911}, "restart_count": 18, "restart_count_per_hour": 11, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.104, "sample_count": 1823, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 230719, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:31:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T21:31:39+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T21:31:39+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T21:31:39.898986+09:00", "mtime_epoch": 1782909099.8989856, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102007}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:31:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 974.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.752, "pushed": 0.752, "read": 0.81}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.672, "count": 13, "max_ms": 4.358, "p95_ms": 4.358}, "gstreamer_push": {"avg_ms": 1.364, "count": 13, "max_ms": 1.457, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.776, "count": 13, "max_ms": 76.977, "p95_ms": 76.977}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.683, "count": 13, "max_ms": 76.964, "p95_ms": 76.964}, "yolo11_640_fire_11class": {"avg_ms": 67.344, "count": 13, "max_ms": 77.469, "p95_ms": 77.469}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.106, "count": 13, "max_ms": 86.454, "p95_ms": 86.454}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.989, "count": 13, "max_ms": 101.364, "p95_ms": 101.364}, "read_interval": {"avg_ms": 2467.756, "count": 14, "max_ms": 17336.568, "p95_ms": 17336.568}, "triton_wait": {"avg_ms": 73.262, "count": 13, "max_ms": 86.467, "p95_ms": 86.467}, "visualize": {"avg_ms": 3.719, "count": 13, "max_ms": 3.977, "p95_ms": 3.977}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.277, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1922}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4063.219, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:31:39+09:00", "last_error_summary": "[h264 @ 0x55f464b68900] error while decoding MB 0 57, bytestream -11 [in#0/rtsp @ 0x55f4645d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f4645d6e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:31:39+09:00", "last_error_summary": "[h264 @ 0x55f464b68900] error while decoding MB 0 57, bytestream -11 [in#0/rtsp @ 0x55f4645d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f4645d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:31:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.108, "last_started_at": "2026-07-01T21:29:23+09:00", "pid": 24911, "process": {"exists": false, "pid": 24911}, "restart_count": 18, "restart_count_per_hour": 11, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.104, "sample_count": 1824, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 230720, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:31:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T21:31:39+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-01T21:31:39+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-01T21:31:39.898986+09:00", "mtime_epoch": 1782909099.8989856, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102007}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:31:40+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 974.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.808, "pushed": 0.808, "read": 0.808}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.663, "count": 14, "max_ms": 4.358, "p95_ms": 4.358}, "gstreamer_push": {"avg_ms": 1.362, "count": 14, "max_ms": 1.457, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.914, "count": 14, "max_ms": 76.977, "p95_ms": 76.977}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.43, "count": 14, "max_ms": 78.129, "p95_ms": 78.129}, "yolo11_640_fire_11class": {"avg_ms": 68.151, "count": 14, "max_ms": 78.644, "p95_ms": 78.644}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.025, "count": 14, "max_ms": 86.454, "p95_ms": 86.454}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.804, "count": 14, "max_ms": 101.364, "p95_ms": 101.364}, "read_interval": {"avg_ms": 2467.756, "count": 14, "max_ms": 17336.568, "p95_ms": 17336.568}, "triton_wait": {"avg_ms": 74.171, "count": 14, "max_ms": 86.467, "p95_ms": 86.467}, "visualize": {"avg_ms": 3.722, "count": 14, "max_ms": 3.977, "p95_ms": 3.977}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.318, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1922}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4063.219, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:31:39+09:00", "last_error_summary": "[h264 @ 0x55f464b68900] error while decoding MB 0 57, bytestream -11 [in#0/rtsp @ 0x55f4645d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f4645d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:31:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.115, "last_started_at": "2026-07-01T21:29:23+09:00", "pid": 24911, "process": {"exists": false, "pid": 24911}, "restart_count": 18, "restart_count_per_hour": 11, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.104, "sample_count": 1824, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 230720, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:31:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T21:31:39+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-01T21:31:39+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T21:31:39.898986+09:00", "mtime_epoch": 1782909099.8989856, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102007}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:31:40+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 974.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.808, "pushed": 0.808, "read": 0.808}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.663, "count": 14, "max_ms": 4.358, "p95_ms": 4.358}, "gstreamer_push": {"avg_ms": 1.362, "count": 14, "max_ms": 1.457, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.914, "count": 14, "max_ms": 76.977, "p95_ms": 76.977}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.43, "count": 14, "max_ms": 78.129, "p95_ms": 78.129}, "yolo11_640_fire_11class": {"avg_ms": 68.151, "count": 14, "max_ms": 78.644, "p95_ms": 78.644}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.025, "count": 14, "max_ms": 86.454, "p95_ms": 86.454}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.804, "count": 14, "max_ms": 101.364, "p95_ms": 101.364}, "read_interval": {"avg_ms": 2467.756, "count": 14, "max_ms": 17336.568, "p95_ms": 17336.568}, "triton_wait": {"avg_ms": 74.171, "count": 14, "max_ms": 86.467, "p95_ms": 86.467}, "visualize": {"avg_ms": 3.722, "count": 14, "max_ms": 3.977, "p95_ms": 3.977}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.324, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 1922}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4081.289, "limit_mb": 8796093022207.996}, "detail": {"pid": 25063, "restart_count": 19, "start_count": 20}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:31:39+09:00", "last_error_summary": "[h264 @ 0x55f464b68900] error while decoding MB 0 57, bytestream -11 [in#0/rtsp @ 0x55f4645d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f4645d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:31:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:31:42+09:00", "pid": 25063, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.031, "name": "ffmpeg", "pid": 25063, "rss_mb": 58.031, "state": "S (sleeping)", "threads": 1}, "restart_count": 19, "restart_count_per_hour": 12, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.104, "sample_count": 1824, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 230720, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:31:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.318, "last_frame_pushed_at": "2026-07-01T21:31:39+09:00", "last_frame_read_age_sec": 4.418, "last_frame_read_at": "2026-07-01T21:31:39+09:00", "mjpeg": {"age_sec": 4.302, "exists": true, "mtime": "2026-07-01T21:31:40.010988+09:00", "mtime_epoch": 1782909100.0109875, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94694}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:31:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 974.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.647, "pushed": 0.647, "read": 0.647}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.663, "count": 14, "max_ms": 4.358, "p95_ms": 4.358}, "gstreamer_push": {"avg_ms": 1.362, "count": 14, "max_ms": 1.457, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.914, "count": 14, "max_ms": 76.977, "p95_ms": 76.977}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.43, "count": 14, "max_ms": 78.129, "p95_ms": 78.129}, "yolo11_640_fire_11class": {"avg_ms": 68.151, "count": 14, "max_ms": 78.644, "p95_ms": 78.644}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.025, "count": 14, "max_ms": 86.454, "p95_ms": 86.454}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.804, "count": 14, "max_ms": 101.364, "p95_ms": 101.364}, "read_interval": {"avg_ms": 2467.756, "count": 14, "max_ms": 17336.568, "p95_ms": 17336.568}, "triton_wait": {"avg_ms": 74.171, "count": 14, "max_ms": 86.467, "p95_ms": 86.467}, "visualize": {"avg_ms": 3.722, "count": 14, "max_ms": 3.977, "p95_ms": 3.977}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.631, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 999}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4102.562, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56206dd8f800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206dd8f800] error while decoding MB 0 8, bytestream 3624 [h264 @ 0x56206d008840] left block unavailable for requested intra mode [h264 @ 0x56206d008840] error while decoding MB 0 6, bytestream 6418 [h264 @ 0x56206d08f580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206d08f580] error while decoding MB 0 32, bytestream 1251 [in#0/rtsp @ 0x56206cfd9e00] Error during demuxing: Connection t", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 159, "h264_decode_error_count": 178, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:39:23+09:00", "last_error_summary": "[h264 @ 0x56206dd8f800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206dd8f800] error while decoding MB 0 8, bytestream 3624 [h264 @ 0x56206d008840] left block unavailable for requested intra mode [h264 @ 0x56206d008840] error while decoding MB 0 6, bytestream 6418 [h264 @ 0x56206d08f580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206d08f580] error while decoding MB 0 32, bytestream 1251 [in#0/rtsp @ 0x56206cfd9e00] Error during demuxing: Connection t", "last_exit_at": "2026-07-01T21:19:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 1167.06, "last_started_at": "2026-07-01T21:19:56+09:00", "pid": 24348, "process": {"exists": false, "pid": 24348}, "restart_count": 183, "restart_count_per_hour": 6, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 184}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.71, "sample_count": 2555, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 119975, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:39:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-01T21:39:23+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-01T21:39:23+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-01T21:39:23.919449+09:00", "mtime_epoch": 1782909563.9194489, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122709}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:39:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 920.434, "name": "celery", "pid": 237, "rss_mb": 915.277, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 103, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.401, "pushed": 3.401, "read": 3.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.556, "count": 102, "max_ms": 2.229, "p95_ms": 1.782}, "gstreamer_push": {"avg_ms": 1.623, "count": 102, "max_ms": 6.307, "p95_ms": 3.858}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.725, "count": 102, "max_ms": 78.145, "p95_ms": 74.557}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.123, "count": 102, "max_ms": 83.372, "p95_ms": 78.307}, "yolo11_640_fire_11class": {"avg_ms": 71.764, "count": 102, "max_ms": 77.476, "p95_ms": 76.386}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.188, "count": 102, "max_ms": 87.115, "p95_ms": 85.926}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.771, "count": 102, "max_ms": 97.478, "p95_ms": 97.084}, "read_interval": {"avg_ms": 291.648, "count": 103, "max_ms": 10126.601, "p95_ms": 115.02}, "triton_wait": {"avg_ms": 80.25, "count": 102, "max_ms": 87.128, "p95_ms": 85.938}, "visualize": {"avg_ms": 3.703, "count": 102, "max_ms": 4.611, "p95_ms": 3.922}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 999}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4102.098, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:39:23+09:00", "last_error_summary": "[h264 @ 0x56206dd8f800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206dd8f800] error while decoding MB 0 8, bytestream 3624 [h264 @ 0x56206d008840] left block unavailable for requested intra mode [h264 @ 0x56206d008840] error while decoding MB 0 6, bytestream 6418 [h264 @ 0x56206d08f580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206d08f580] error while decoding MB 0 32, bytestream 1251 [in#0/rtsp @ 0x56206cfd9e00] Error during demuxing: Connection t", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 159, "h264_decode_error_count": 178, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:39:23+09:00", "last_error_summary": "[h264 @ 0x56206dd8f800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206dd8f800] error while decoding MB 0 8, bytestream 3624 [h264 @ 0x56206d008840] left block unavailable for requested intra mode [h264 @ 0x56206d008840] error while decoding MB 0 6, bytestream 6418 [h264 @ 0x56206d08f580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206d08f580] error while decoding MB 0 32, bytestream 1251 [in#0/rtsp @ 0x56206cfd9e00] Error during demuxing: Connection t", "last_exit_at": "2026-07-01T21:39:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 1167.1, "last_started_at": "2026-07-01T21:19:56+09:00", "pid": 24348, "process": {"exists": false, "pid": 24348}, "restart_count": 183, "restart_count_per_hour": 6, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 184}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.71, "sample_count": 2555, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 119976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:39:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T21:39:24+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-01T21:39:23+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-01T21:39:23.919449+09:00", "mtime_epoch": 1782909563.9194489, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122709}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:39:24+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 920.434, "name": "celery", "pid": 237, "rss_mb": 915.277, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.441, "pushed": 3.441, "read": 3.408}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.556, "count": 103, "max_ms": 2.229, "p95_ms": 1.782}, "gstreamer_push": {"avg_ms": 1.62, "count": 103, "max_ms": 6.307, "p95_ms": 3.858}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.681, "count": 103, "max_ms": 78.145, "p95_ms": 74.557}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.182, "count": 103, "max_ms": 83.372, "p95_ms": 78.335}, "yolo11_640_fire_11class": {"avg_ms": 71.788, "count": 103, "max_ms": 77.476, "p95_ms": 76.386}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.19, "count": 103, "max_ms": 87.115, "p95_ms": 85.926}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.771, "count": 103, "max_ms": 97.478, "p95_ms": 97.084}, "read_interval": {"avg_ms": 293.412, "count": 102, "max_ms": 10126.601, "p95_ms": 115.02}, "triton_wait": {"avg_ms": 80.251, "count": 103, "max_ms": 87.128, "p95_ms": 85.938}, "visualize": {"avg_ms": 3.703, "count": 103, "max_ms": 4.611, "p95_ms": 3.922}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 999}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4101.758, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 159, "h264_decode_error_count": 178, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:39:23+09:00", "last_error_summary": "[h264 @ 0x56206dd8f800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206dd8f800] error while decoding MB 0 8, bytestream 3624 [h264 @ 0x56206d008840] left block unavailable for requested intra mode [h264 @ 0x56206d008840] error while decoding MB 0 6, bytestream 6418 [h264 @ 0x56206d08f580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206d08f580] error while decoding MB 0 32, bytestream 1251 [in#0/rtsp @ 0x56206cfd9e00] Error during demuxing: Connection t", "last_exit_at": "2026-07-01T21:39:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 1167.108, "last_started_at": "2026-07-01T21:19:56+09:00", "pid": 24348, "process": {"exists": false, "pid": 24348}, "restart_count": 183, "restart_count_per_hour": 6, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 184}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.71, "sample_count": 2555, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 119976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:39:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T21:39:24+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-01T21:39:23+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-01T21:39:23.919449+09:00", "mtime_epoch": 1782909563.9194489, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122709}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:39:24+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 920.434, "name": "celery", "pid": 237, "rss_mb": 915.277, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.44, "pushed": 3.44, "read": 3.407}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.556, "count": 103, "max_ms": 2.229, "p95_ms": 1.782}, "gstreamer_push": {"avg_ms": 1.62, "count": 103, "max_ms": 6.307, "p95_ms": 3.858}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.681, "count": 103, "max_ms": 78.145, "p95_ms": 74.557}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.182, "count": 103, "max_ms": 83.372, "p95_ms": 78.335}, "yolo11_640_fire_11class": {"avg_ms": 71.788, "count": 103, "max_ms": 77.476, "p95_ms": 76.386}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.19, "count": 103, "max_ms": 87.115, "p95_ms": 85.926}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.771, "count": 103, "max_ms": 97.478, "p95_ms": 97.084}, "read_interval": {"avg_ms": 293.412, "count": 102, "max_ms": 10126.601, "p95_ms": 115.02}, "triton_wait": {"avg_ms": 80.251, "count": 103, "max_ms": 87.128, "p95_ms": 85.938}, "visualize": {"avg_ms": 3.703, "count": 103, "max_ms": 4.611, "p95_ms": 3.922}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 999}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4214.895, "limit_mb": 8796093022207.996}, "detail": {"pid": 25215, "restart_count": 184, "start_count": 185}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 159, "h264_decode_error_count": 178, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:39:23+09:00", "last_error_summary": "[h264 @ 0x56206dd8f800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206dd8f800] error while decoding MB 0 8, bytestream 3624 [h264 @ 0x56206d008840] left block unavailable for requested intra mode [h264 @ 0x56206d008840] error while decoding MB 0 6, bytestream 6418 [h264 @ 0x56206d08f580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56206d08f580] error while decoding MB 0 32, bytestream 1251 [in#0/rtsp @ 0x56206cfd9e00] Error during demuxing: Connection t", "last_exit_at": "2026-07-01T21:39:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:39:26+09:00", "pid": 25215, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.375, "name": "ffmpeg", "pid": 25215, "rss_mb": 138.375, "state": "S (sleeping)", "threads": 106}, "restart_count": 184, "restart_count_per_hour": 7, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 185}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.658, "sample_count": 2518, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 119976, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:39:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.168, "last_frame_pushed_at": "2026-07-01T21:39:24+09:00", "last_frame_read_age_sec": 4.261, "last_frame_read_at": "2026-07-01T21:39:23+09:00", "mjpeg": {"age_sec": 4.152, "exists": true, "mtime": "2026-07-01T21:39:24.023451+09:00", "mtime_epoch": 1782909564.0234509, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123731}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:39:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 920.434, "name": "celery", "pid": 237, "rss_mb": 915.535, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.173, "pushed": 2.173, "read": 2.139}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.555, "count": 65, "max_ms": 2.101, "p95_ms": 1.782}, "gstreamer_push": {"avg_ms": 1.586, "count": 65, "max_ms": 3.968, "p95_ms": 3.781}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.538, "count": 65, "max_ms": 78.145, "p95_ms": 75.684}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.238, "count": 65, "max_ms": 81.929, "p95_ms": 78.335}, "yolo11_640_fire_11class": {"avg_ms": 70.743, "count": 65, "max_ms": 77.476, "p95_ms": 76.848}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.678, "count": 65, "max_ms": 87.115, "p95_ms": 86.772}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.284, "count": 65, "max_ms": 97.478, "p95_ms": 97.114}, "read_interval": {"avg_ms": 402.303, "count": 64, "max_ms": 10126.601, "p95_ms": 115.123}, "triton_wait": {"avg_ms": 78.767, "count": 65, "max_ms": 87.128, "p95_ms": 86.785}, "visualize": {"avg_ms": 3.707, "count": 65, "max_ms": 4.014, "p95_ms": 3.922}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1053}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4114.527, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557b60370800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b60370800] error while decoding MB 0 43, bytestream 1175 [h264 @ 0x557b5f642840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b5f642840] error while decoding MB 0 39, bytestream 1952 [h264 @ 0x557b60370800] left block unavailable for requested intra mode [h264 @ 0x557b60370800] error while decoding MB 0 39, bytestream 1646 [h264 @ 0x557b60370800] left block unavailable for requested i", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 160, "h264_decode_error_count": 182, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:51:27+09:00", "last_error_summary": "[h264 @ 0x557b60370800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b60370800] error while decoding MB 0 43, bytestream 1175 [h264 @ 0x557b5f642840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b5f642840] error while decoding MB 0 39, bytestream 1952 [h264 @ 0x557b60370800] left block unavailable for requested intra mode [h264 @ 0x557b60370800] error while decoding MB 0 39, bytestream 1646 [h264 @ 0x557b60370800] left block unavailable for requested i", "last_exit_at": "2026-07-01T21:39:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 721.784, "last_started_at": "2026-07-01T21:39:26+09:00", "pid": 25215, "process": {"exists": false, "pid": 25215}, "restart_count": 184, "restart_count_per_hour": 4, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 185}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.863, "sample_count": 2561, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 126461, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:51:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T21:51:27+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-01T21:51:27+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-01T21:51:27.897707+09:00", "mtime_epoch": 1782910287.897707, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127822}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:51:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 923.602, "name": "celery", "pid": 237, "rss_mb": 918.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.34, "pushed": 3.34, "read": 3.34}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.517, "count": 100, "max_ms": 1.989, "p95_ms": 1.55}, "gstreamer_push": {"avg_ms": 1.46, "count": 100, "max_ms": 6.145, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.129, "count": 100, "max_ms": 83.18, "p95_ms": 75.673}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.871, "count": 100, "max_ms": 84.237, "p95_ms": 78.874}, "yolo11_640_fire_11class": {"avg_ms": 72.183, "count": 100, "max_ms": 83.784, "p95_ms": 78.518}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.815, "count": 100, "max_ms": 93.283, "p95_ms": 85.929}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.173, "count": 100, "max_ms": 103.634, "p95_ms": 97.006}, "read_interval": {"avg_ms": 299.729, "count": 100, "max_ms": 19435.136, "p95_ms": 115.526}, "triton_wait": {"avg_ms": 79.833, "count": 100, "max_ms": 93.298, "p95_ms": 85.941}, "visualize": {"avg_ms": 3.696, "count": 100, "max_ms": 4.658, "p95_ms": 3.906}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1053}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4113.941, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:51:27+09:00", "last_error_summary": "[h264 @ 0x557b60370800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b60370800] error while decoding MB 0 43, bytestream 1175 [h264 @ 0x557b5f642840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b5f642840] error while decoding MB 0 39, bytestream 1952 [h264 @ 0x557b60370800] left block unavailable for requested intra mode [h264 @ 0x557b60370800] error while decoding MB 0 39, bytestream 1646 [h264 @ 0x557b60370800] left block unavailable for requested i", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 160, "h264_decode_error_count": 182, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:51:27+09:00", "last_error_summary": "[h264 @ 0x557b60370800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b60370800] error while decoding MB 0 43, bytestream 1175 [h264 @ 0x557b5f642840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b5f642840] error while decoding MB 0 39, bytestream 1952 [h264 @ 0x557b60370800] left block unavailable for requested intra mode [h264 @ 0x557b60370800] error while decoding MB 0 39, bytestream 1646 [h264 @ 0x557b60370800] left block unavailable for requested i", "last_exit_at": "2026-07-01T21:51:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 721.829, "last_started_at": "2026-07-01T21:39:26+09:00", "pid": 25215, "process": {"exists": false, "pid": 25215}, "restart_count": 184, "restart_count_per_hour": 4, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 185}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.863, "sample_count": 2561, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 126462, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:51:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T21:51:27+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T21:51:27+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-01T21:51:27.897707+09:00", "mtime_epoch": 1782910287.897707, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127822}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:51:28+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 923.602, "name": "celery", "pid": 237, "rss_mb": 918.504, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.369, "pushed": 3.369, "read": 3.335}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.517, "count": 101, "max_ms": 1.989, "p95_ms": 1.55}, "gstreamer_push": {"avg_ms": 1.458, "count": 101, "max_ms": 6.145, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.226, "count": 101, "max_ms": 83.18, "p95_ms": 75.673}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.914, "count": 101, "max_ms": 84.237, "p95_ms": 78.874}, "yolo11_640_fire_11class": {"avg_ms": 72.22, "count": 101, "max_ms": 83.784, "p95_ms": 78.518}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.866, "count": 101, "max_ms": 93.283, "p95_ms": 85.929}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.224, "count": 101, "max_ms": 103.634, "p95_ms": 97.006}, "read_interval": {"avg_ms": 299.729, "count": 100, "max_ms": 19435.136, "p95_ms": 115.526}, "triton_wait": {"avg_ms": 79.883, "count": 101, "max_ms": 93.298, "p95_ms": 85.941}, "visualize": {"avg_ms": 3.697, "count": 101, "max_ms": 4.658, "p95_ms": 3.906}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1053}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4114.156, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 160, "h264_decode_error_count": 182, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:51:27+09:00", "last_error_summary": "[h264 @ 0x557b60370800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b60370800] error while decoding MB 0 43, bytestream 1175 [h264 @ 0x557b5f642840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b5f642840] error while decoding MB 0 39, bytestream 1952 [h264 @ 0x557b60370800] left block unavailable for requested intra mode [h264 @ 0x557b60370800] error while decoding MB 0 39, bytestream 1646 [h264 @ 0x557b60370800] left block unavailable for requested i", "last_exit_at": "2026-07-01T21:51:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 721.837, "last_started_at": "2026-07-01T21:39:26+09:00", "pid": 25215, "process": {"exists": false, "pid": 25215}, "restart_count": 184, "restart_count_per_hour": 4, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 185}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.863, "sample_count": 2561, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 126462, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:51:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T21:51:27+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-01T21:51:27+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-01T21:51:27.897707+09:00", "mtime_epoch": 1782910287.897707, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127822}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:51:28+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 923.602, "name": "celery", "pid": 237, "rss_mb": 918.504, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.368, "pushed": 3.368, "read": 3.335}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.517, "count": 101, "max_ms": 1.989, "p95_ms": 1.55}, "gstreamer_push": {"avg_ms": 1.458, "count": 101, "max_ms": 6.145, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.226, "count": 101, "max_ms": 83.18, "p95_ms": 75.673}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.914, "count": 101, "max_ms": 84.237, "p95_ms": 78.874}, "yolo11_640_fire_11class": {"avg_ms": 72.22, "count": 101, "max_ms": 83.784, "p95_ms": 78.518}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.866, "count": 101, "max_ms": 93.283, "p95_ms": 85.929}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.224, "count": 101, "max_ms": 103.634, "p95_ms": 97.006}, "read_interval": {"avg_ms": 299.729, "count": 100, "max_ms": 19435.136, "p95_ms": 115.526}, "triton_wait": {"avg_ms": 79.883, "count": 101, "max_ms": 93.298, "p95_ms": 85.941}, "visualize": {"avg_ms": 3.697, "count": 101, "max_ms": 4.658, "p95_ms": 3.906}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1053}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4219.172, "limit_mb": 8796093022207.996}, "detail": {"pid": 25322, "restart_count": 185, "start_count": 186}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 160, "h264_decode_error_count": 182, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:51:27+09:00", "last_error_summary": "[h264 @ 0x557b60370800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b60370800] error while decoding MB 0 43, bytestream 1175 [h264 @ 0x557b5f642840] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b5f642840] error while decoding MB 0 39, bytestream 1952 [h264 @ 0x557b60370800] left block unavailable for requested intra mode [h264 @ 0x557b60370800] error while decoding MB 0 39, bytestream 1646 [h264 @ 0x557b60370800] left block unavailable for requested i", "last_exit_at": "2026-07-01T21:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:51:31+09:00", "pid": 25322, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.012, "name": "ffmpeg", "pid": 25322, "rss_mb": 139.012, "state": "S (sleeping)", "threads": 106}, "restart_count": 185, "restart_count_per_hour": 5, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 186}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.877, "sample_count": 2515, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 126462, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:51:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 5.014, "last_frame_pushed_at": "2026-07-01T21:51:27+09:00", "last_frame_read_age_sec": 5.111, "last_frame_read_at": "2026-07-01T21:51:27+09:00", "mjpeg": {"age_sec": 4.996, "exists": true, "mtime": "2026-07-01T21:51:28.013708+09:00", "mtime_epoch": 1782910288.0137076, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129413}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T21:51:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 923.602, "name": "celery", "pid": 237, "rss_mb": 918.504, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.837, "pushed": 1.837, "read": 1.804}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.513, "count": 55, "max_ms": 1.989, "p95_ms": 1.634}, "gstreamer_push": {"avg_ms": 1.352, "count": 55, "max_ms": 1.446, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.294, "count": 55, "max_ms": 83.18, "p95_ms": 78.431}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.15, "count": 55, "max_ms": 84.237, "p95_ms": 80.247}, "yolo11_640_fire_11class": {"avg_ms": 70.523, "count": 55, "max_ms": 83.784, "p95_ms": 78.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.487, "count": 55, "max_ms": 93.283, "p95_ms": 86.751}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.768, "count": 55, "max_ms": 103.634, "p95_ms": 97.335}, "read_interval": {"avg_ms": 461.512, "count": 54, "max_ms": 19435.136, "p95_ms": 115.645}, "triton_wait": {"avg_ms": 77.509, "count": 55, "max_ms": 93.298, "p95_ms": 86.763}, "visualize": {"avg_ms": 3.672, "count": 55, "max_ms": 4.35, "p95_ms": 3.837}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2031}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4050.805, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[NULL @ 0x561a6e0e2200] missing picture in access unit with size 18 [h264 @ 0x561a6edb8cc0] no frame! [vist#0:0/h264 @ 0x561a6e0e2840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:55:19+09:00", "last_error_summary": "[NULL @ 0x561a6e0e2200] missing picture in access unit with size 18 [h264 @ 0x561a6edb8cc0] no frame! [vist#0:0/h264 @ 0x561a6e0e2840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:31:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 1417.007, "last_started_at": "2026-07-01T21:31:42+09:00", "pid": 25063, "process": {"exists": false, "pid": 25063}, "restart_count": 19, "restart_count_per_hour": 12, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.89, "sample_count": 2851, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 243797, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:55:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T21:55:19+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-01T21:55:19+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T21:55:19.259499+09:00", "mtime_epoch": 1782910519.2594988, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98376}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:55:19+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 980.473, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 272, "pushed_frames": 272, "read_frames": 273, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.068, "pushed": 9.068, "read": 9.102}, "hints": ["triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.658, "count": 272, "max_ms": 6.227, "p95_ms": 4.035}, "gstreamer_push": {"avg_ms": 1.609, "count": 272, "max_ms": 6.772, "p95_ms": 3.847}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.029, "count": 272, "max_ms": 80.552, "p95_ms": 76.059}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.881, "count": 272, "max_ms": 81.546, "p95_ms": 77.063}, "yolo11_640_fire_11class": {"avg_ms": 71.92, "count": 272, "max_ms": 81.537, "p95_ms": 76.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.296, "count": 272, "max_ms": 87.784, "p95_ms": 84.844}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.983, "count": 272, "max_ms": 100.324, "p95_ms": 97.58}, "read_interval": {"avg_ms": 110.04, "count": 273, "max_ms": 119.61, "p95_ms": 115.604}, "triton_wait": {"avg_ms": 80.309, "count": 272, "max_ms": 87.797, "p95_ms": 84.859}, "visualize": {"avg_ms": 3.701, "count": 272, "max_ms": 4.658, "p95_ms": 4.045}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2031}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4053.059, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:55:19+09:00", "last_error_summary": "[NULL @ 0x561a6e0e2200] missing picture in access unit with size 18 [h264 @ 0x561a6edb8cc0] no frame! [vist#0:0/h264 @ 0x561a6e0e2840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:55:19+09:00", "last_error_summary": "[NULL @ 0x561a6e0e2200] missing picture in access unit with size 18 [h264 @ 0x561a6edb8cc0] no frame! [vist#0:0/h264 @ 0x561a6e0e2840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:55:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 1417.046, "last_started_at": "2026-07-01T21:31:42+09:00", "pid": 25063, "process": {"exists": false, "pid": 25063}, "restart_count": 19, "restart_count_per_hour": 12, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.89, "sample_count": 2851, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 243798, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:55:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T21:55:19+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T21:55:19+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-01T21:55:19.259499+09:00", "mtime_epoch": 1782910519.2594988, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98376}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:55:19+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 980.473, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 273, "pushed_frames": 273, "read_frames": 272, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.118, "pushed": 9.118, "read": 9.084}, "hints": ["triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.661, "count": 273, "max_ms": 6.227, "p95_ms": 4.262}, "gstreamer_push": {"avg_ms": 1.608, "count": 273, "max_ms": 6.772, "p95_ms": 3.847}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.034, "count": 273, "max_ms": 80.552, "p95_ms": 76.059}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.892, "count": 273, "max_ms": 81.546, "p95_ms": 77.063}, "yolo11_640_fire_11class": {"avg_ms": 71.914, "count": 273, "max_ms": 81.537, "p95_ms": 76.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.282, "count": 273, "max_ms": 87.784, "p95_ms": 84.844}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.972, "count": 273, "max_ms": 100.324, "p95_ms": 97.58}, "read_interval": {"avg_ms": 110.049, "count": 272, "max_ms": 119.61, "p95_ms": 115.604}, "triton_wait": {"avg_ms": 80.295, "count": 273, "max_ms": 87.797, "p95_ms": 84.859}, "visualize": {"avg_ms": 3.701, "count": 273, "max_ms": 4.658, "p95_ms": 4.045}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2031}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4052.969, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:55:19+09:00", "last_error_summary": "[NULL @ 0x561a6e0e2200] missing picture in access unit with size 18 [h264 @ 0x561a6edb8cc0] no frame! [vist#0:0/h264 @ 0x561a6e0e2840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:55:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 1417.055, "last_started_at": "2026-07-01T21:31:42+09:00", "pid": 25063, "process": {"exists": false, "pid": 25063}, "restart_count": 19, "restart_count_per_hour": 12, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.89, "sample_count": 2851, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 243798, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:55:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-01T21:55:19+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-01T21:55:19+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T21:55:19.259499+09:00", "mtime_epoch": 1782910519.2594988, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98376}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:55:19+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 980.473, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 273, "pushed_frames": 273, "read_frames": 272, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.115, "pushed": 9.115, "read": 9.082}, "hints": ["triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.661, "count": 273, "max_ms": 6.227, "p95_ms": 4.262}, "gstreamer_push": {"avg_ms": 1.608, "count": 273, "max_ms": 6.772, "p95_ms": 3.847}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.034, "count": 273, "max_ms": 80.552, "p95_ms": 76.059}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.892, "count": 273, "max_ms": 81.546, "p95_ms": 77.063}, "yolo11_640_fire_11class": {"avg_ms": 71.914, "count": 273, "max_ms": 81.537, "p95_ms": 76.36}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.282, "count": 273, "max_ms": 87.784, "p95_ms": 84.844}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.972, "count": 273, "max_ms": 100.324, "p95_ms": 97.58}, "read_interval": {"avg_ms": 110.049, "count": 272, "max_ms": 119.61, "p95_ms": 115.604}, "triton_wait": {"avg_ms": 80.295, "count": 273, "max_ms": 87.797, "p95_ms": 84.859}, "visualize": {"avg_ms": 3.701, "count": 273, "max_ms": 4.658, "p95_ms": 4.045}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2031}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4074.02, "limit_mb": 8796093022207.996}, "detail": {"pid": 25429, "restart_count": 20, "start_count": 21}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:55:19+09:00", "last_error_summary": "[NULL @ 0x561a6e0e2200] missing picture in access unit with size 18 [h264 @ 0x561a6edb8cc0] no frame! [vist#0:0/h264 @ 0x561a6e0e2840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:55:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:55:22+09:00", "pid": 25429, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.359, "name": "ffmpeg", "pid": 25429, "rss_mb": 57.359, "state": "S (sleeping)", "threads": 1}, "restart_count": 20, "restart_count_per_hour": 13, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.89, "sample_count": 2806, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 243798, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:55:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.946, "last_frame_pushed_at": "2026-07-01T21:55:19+09:00", "last_frame_read_age_sec": 5.038, "last_frame_read_at": "2026-07-01T21:55:19+09:00", "mjpeg": {"age_sec": 4.928, "exists": true, "mtime": "2026-07-01T21:55:19.367500+09:00", "mtime_epoch": 1782910519.3674998, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99133}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:55:24+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 980.473, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 228, "pushed_frames": 228, "read_frames": 227, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.62, "pushed": 7.62, "read": 7.586}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.673, "count": 228, "max_ms": 6.227, "p95_ms": 4.316}, "gstreamer_push": {"avg_ms": 1.641, "count": 228, "max_ms": 6.772, "p95_ms": 3.853}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 69.274, "count": 228, "max_ms": 80.552, "p95_ms": 76.059}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.843, "count": 228, "max_ms": 81.546, "p95_ms": 77.446}, "yolo11_640_fire_11class": {"avg_ms": 71.832, "count": 228, "max_ms": 81.537, "p95_ms": 76.455}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.162, "count": 228, "max_ms": 87.784, "p95_ms": 85.17}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.902, "count": 228, "max_ms": 100.324, "p95_ms": 98.128}, "read_interval": {"avg_ms": 110.043, "count": 227, "max_ms": 119.61, "p95_ms": 115.725}, "triton_wait": {"avg_ms": 80.174, "count": 228, "max_ms": 87.797, "p95_ms": 85.182}, "visualize": {"avg_ms": 3.71, "count": 228, "max_ms": 4.658, "p95_ms": 4.062}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2044}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4060.727, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5611cca9b480] error while decoding MB 53 21, bytestream -5 [in#0/rtsp @ 0x5611cbcfee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611cbcfee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:58:37+09:00", "last_error_summary": "[h264 @ 0x5611cca9b480] error while decoding MB 53 21, bytestream -5 [in#0/rtsp @ 0x5611cbcfee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611cbcfee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:55:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.921, "last_started_at": "2026-07-01T21:55:22+09:00", "pid": 25429, "process": {"exists": false, "pid": 25429}, "restart_count": 20, "restart_count_per_hour": 12, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.72, "sample_count": 2412, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245289, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:58:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T21:58:37+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T21:58:37+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-01T21:58:37.149593+09:00", "mtime_epoch": 1782910717.1495926, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 88924}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:58:37+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 980.848, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 26, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.867, "pushed": 0.867, "read": 0.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.698, "count": 26, "max_ms": 4.617, "p95_ms": 4.12}, "gstreamer_push": {"avg_ms": 1.537, "count": 26, "max_ms": 4.076, "p95_ms": 3.733}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.041, "count": 26, "max_ms": 75.442, "p95_ms": 74.205}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.011, "count": 26, "max_ms": 82.86, "p95_ms": 76.512}, "yolo11_640_fire_11class": {"avg_ms": 72.909, "count": 26, "max_ms": 76.502, "p95_ms": 75.237}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.567, "count": 26, "max_ms": 84.401, "p95_ms": 83.946}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.268, "count": 26, "max_ms": 96.938, "p95_ms": 96.585}, "read_interval": {"avg_ms": 1154.685, "count": 26, "max_ms": 27268.549, "p95_ms": 116.429}, "triton_wait": {"avg_ms": 81.58, "count": 26, "max_ms": 84.413, "p95_ms": 83.959}, "visualize": {"avg_ms": 3.706, "count": 26, "max_ms": 3.88, "p95_ms": 3.871}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2044}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4060.273, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T21:58:37+09:00", "last_error_summary": "[h264 @ 0x5611cca9b480] error while decoding MB 53 21, bytestream -5 [in#0/rtsp @ 0x5611cbcfee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611cbcfee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:58:37+09:00", "last_error_summary": "[h264 @ 0x5611cca9b480] error while decoding MB 53 21, bytestream -5 [in#0/rtsp @ 0x5611cbcfee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611cbcfee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:58:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.961, "last_started_at": "2026-07-01T21:55:22+09:00", "pid": 25429, "process": {"exists": false, "pid": 25429}, "restart_count": 20, "restart_count_per_hour": 12, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.72, "sample_count": 2412, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245290, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:58:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T21:58:37+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-01T21:58:37+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-01T21:58:37.149593+09:00", "mtime_epoch": 1782910717.1495926, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 88924}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:58:37+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 980.848, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 25, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.869, "pushed": 0.869, "read": 0.835}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.694, "count": 26, "max_ms": 4.617, "p95_ms": 4.12}, "gstreamer_push": {"avg_ms": 1.544, "count": 26, "max_ms": 4.076, "p95_ms": 3.733}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.917, "count": 26, "max_ms": 75.442, "p95_ms": 74.205}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.11, "count": 26, "max_ms": 82.86, "p95_ms": 76.542}, "yolo11_640_fire_11class": {"avg_ms": 73.081, "count": 26, "max_ms": 76.554, "p95_ms": 76.502}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.685, "count": 26, "max_ms": 85.95, "p95_ms": 84.401}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.387, "count": 26, "max_ms": 98.372, "p95_ms": 96.938}, "read_interval": {"avg_ms": 1196.434, "count": 25, "max_ms": 27268.549, "p95_ms": 116.429}, "triton_wait": {"avg_ms": 81.698, "count": 26, "max_ms": 85.962, "p95_ms": 84.413}, "visualize": {"avg_ms": 3.704, "count": 26, "max_ms": 3.88, "p95_ms": 3.871}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2044}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4060.172, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:58:37+09:00", "last_error_summary": "[h264 @ 0x5611cca9b480] error while decoding MB 53 21, bytestream -5 [in#0/rtsp @ 0x5611cbcfee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611cbcfee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:58:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.969, "last_started_at": "2026-07-01T21:55:22+09:00", "pid": 25429, "process": {"exists": false, "pid": 25429}, "restart_count": 20, "restart_count_per_hour": 12, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.72, "sample_count": 2412, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245290, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:58:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T21:58:37+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-01T21:58:37+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-01T21:58:37.149593+09:00", "mtime_epoch": 1782910717.1495926, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 88924}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:58:37+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 980.848, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 25, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.869, "pushed": 0.869, "read": 0.835}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.694, "count": 26, "max_ms": 4.617, "p95_ms": 4.12}, "gstreamer_push": {"avg_ms": 1.544, "count": 26, "max_ms": 4.076, "p95_ms": 3.733}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.917, "count": 26, "max_ms": 75.442, "p95_ms": 74.205}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.11, "count": 26, "max_ms": 82.86, "p95_ms": 76.542}, "yolo11_640_fire_11class": {"avg_ms": 73.081, "count": 26, "max_ms": 76.554, "p95_ms": 76.502}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.685, "count": 26, "max_ms": 85.95, "p95_ms": 84.401}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.387, "count": 26, "max_ms": 98.372, "p95_ms": 96.938}, "read_interval": {"avg_ms": 1196.434, "count": 25, "max_ms": 27268.549, "p95_ms": 116.429}, "triton_wait": {"avg_ms": 81.698, "count": 26, "max_ms": 85.962, "p95_ms": 84.413}, "visualize": {"avg_ms": 3.704, "count": 26, "max_ms": 3.88, "p95_ms": 3.871}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2044}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4078.066, "limit_mb": 8796093022207.996}, "detail": {"pid": 25581, "restart_count": 21, "start_count": 22}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T21:58:37+09:00", "last_error_summary": "[h264 @ 0x5611cca9b480] error while decoding MB 53 21, bytestream -5 [in#0/rtsp @ 0x5611cbcfee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611cbcfee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T21:58:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T21:58:40+09:00", "pid": 25581, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.676, "name": "ffmpeg", "pid": 25581, "rss_mb": 57.676, "state": "S (sleeping)", "threads": 1}, "restart_count": 21, "restart_count_per_hour": 13, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.744, "sample_count": 2367, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245290, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T21:58:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.989, "last_frame_pushed_at": "2026-07-01T21:58:37+09:00", "last_frame_read_age_sec": 5.089, "last_frame_read_at": "2026-07-01T21:58:37+09:00", "mjpeg": {"age_sec": 4.973, "exists": true, "mtime": "2026-07-01T21:58:37.265594+09:00", "mtime_epoch": 1782910717.265594, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 89024}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T21:58:42+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 980.848, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.028, "pushed": 2.028, "read": 2.028}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.696, "count": 13, "max_ms": 4.12, "p95_ms": 4.12}, "gstreamer_push": {"avg_ms": 1.756, "count": 13, "max_ms": 4.076, "p95_ms": 4.076}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.582, "count": 13, "max_ms": 74.205, "p95_ms": 74.205}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.884, "count": 13, "max_ms": 82.86, "p95_ms": 82.86}, "yolo11_640_fire_11class": {"avg_ms": 73.674, "count": 13, "max_ms": 76.554, "p95_ms": 76.554}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.231, "count": 13, "max_ms": 85.95, "p95_ms": 85.95}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.114, "count": 13, "max_ms": 98.372, "p95_ms": 98.372}, "read_interval": {"avg_ms": 2199.133, "count": 13, "max_ms": 27268.549, "p95_ms": 27268.549}, "triton_wait": {"avg_ms": 82.243, "count": 13, "max_ms": 85.962, "p95_ms": 85.962}, "visualize": {"avg_ms": 3.719, "count": 13, "max_ms": 3.88, "p95_ms": 3.88}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 6.41, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4116.27, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5649d50bb240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d50bb240] error while decoding MB 0 30, bytestream 2651 [h264 @ 0x5649d5bea980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d5bea980] error while decoding MB 0 33, bytestream 5226 [h264 @ 0x5649d51d1b80] cabac decode of qscale diff failed at 74 30 [h264 @ 0x5649d51d1b80] error while decoding MB 74 30, bytestream 1426 [h264 @ 0x5649d5bea980] left block unavailable for requested intr", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 161, "h264_decode_error_count": 191, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:01:49+09:00", "last_error_summary": "[h264 @ 0x5649d50bb240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d50bb240] error while decoding MB 0 30, bytestream 2651 [h264 @ 0x5649d5bea980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d5bea980] error while decoding MB 0 33, bytestream 5226 [h264 @ 0x5649d51d1b80] cabac decode of qscale diff failed at 74 30 [h264 @ 0x5649d51d1b80] error while decoding MB 74 30, bytestream 1426 [h264 @ 0x5649d5bea980] left block unavailable for requested intr", "last_exit_at": "2026-07-01T21:51:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 618.696, "last_started_at": "2026-07-01T21:51:31+09:00", "pid": 25322, "process": {"exists": false, "pid": 25322}, "restart_count": 185, "restart_count_per_hour": 4, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 186}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.293, "sample_count": 2686, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132069, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:01:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-01T22:01:49+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-01T22:01:49+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-01T22:01:49.651960+09:00", "mtime_epoch": 1782910909.65196, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113251}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:01:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 924.613, "name": "celery", "pid": 237, "rss_mb": 919.199, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.341, "pushed": 3.341, "read": 3.341}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.571, "count": 100, "max_ms": 2.046, "p95_ms": 2.017}, "gstreamer_push": {"avg_ms": 1.595, "count": 100, "max_ms": 4.026, "p95_ms": 3.701}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.529, "count": 100, "max_ms": 78.339, "p95_ms": 76.489}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.006, "count": 100, "max_ms": 82.324, "p95_ms": 78.645}, "yolo11_640_fire_11class": {"avg_ms": 73.887, "count": 100, "max_ms": 80.891, "p95_ms": 78.618}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.323, "count": 100, "max_ms": 86.428, "p95_ms": 85.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.092, "count": 100, "max_ms": 99.258, "p95_ms": 96.045}, "read_interval": {"avg_ms": 299.7, "count": 100, "max_ms": 10133.369, "p95_ms": 115.487}, "triton_wait": {"avg_ms": 81.572, "count": 100, "max_ms": 86.44, "p95_ms": 85.707}, "visualize": {"avg_ms": 3.728, "count": 100, "max_ms": 4.936, "p95_ms": 3.979}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4115.598, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:01:49+09:00", "last_error_summary": "[h264 @ 0x5649d50bb240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d50bb240] error while decoding MB 0 30, bytestream 2651 [h264 @ 0x5649d5bea980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d5bea980] error while decoding MB 0 33, bytestream 5226 [h264 @ 0x5649d51d1b80] cabac decode of qscale diff failed at 74 30 [h264 @ 0x5649d51d1b80] error while decoding MB 74 30, bytestream 1426 [h264 @ 0x5649d5bea980] left block unavailable for requested intr", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 161, "h264_decode_error_count": 191, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:01:49+09:00", "last_error_summary": "[h264 @ 0x5649d50bb240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d50bb240] error while decoding MB 0 30, bytestream 2651 [h264 @ 0x5649d5bea980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d5bea980] error while decoding MB 0 33, bytestream 5226 [h264 @ 0x5649d51d1b80] cabac decode of qscale diff failed at 74 30 [h264 @ 0x5649d51d1b80] error while decoding MB 74 30, bytestream 1426 [h264 @ 0x5649d5bea980] left block unavailable for requested intr", "last_exit_at": "2026-07-01T22:01:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 618.742, "last_started_at": "2026-07-01T21:51:31+09:00", "pid": 25322, "process": {"exists": false, "pid": 25322}, "restart_count": 185, "restart_count_per_hour": 4, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 186}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.293, "sample_count": 2686, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132070, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:01:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T22:01:49+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T22:01:49+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-01T22:01:49.651960+09:00", "mtime_epoch": 1782910909.65196, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113251}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:01:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 924.613, "name": "celery", "pid": 237, "rss_mb": 919.199, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.369, "pushed": 3.369, "read": 3.336}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.569, "count": 101, "max_ms": 2.046, "p95_ms": 2.017}, "gstreamer_push": {"avg_ms": 1.593, "count": 101, "max_ms": 4.026, "p95_ms": 3.701}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.567, "count": 101, "max_ms": 78.339, "p95_ms": 76.489}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.013, "count": 101, "max_ms": 82.324, "p95_ms": 78.645}, "yolo11_640_fire_11class": {"avg_ms": 73.903, "count": 101, "max_ms": 80.891, "p95_ms": 78.618}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.341, "count": 101, "max_ms": 86.428, "p95_ms": 85.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.104, "count": 101, "max_ms": 99.258, "p95_ms": 96.045}, "read_interval": {"avg_ms": 299.7, "count": 100, "max_ms": 10133.369, "p95_ms": 115.487}, "triton_wait": {"avg_ms": 81.588, "count": 101, "max_ms": 86.44, "p95_ms": 85.707}, "visualize": {"avg_ms": 3.727, "count": 101, "max_ms": 4.936, "p95_ms": 3.979}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4115.496, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 161, "h264_decode_error_count": 191, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:01:49+09:00", "last_error_summary": "[h264 @ 0x5649d50bb240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d50bb240] error while decoding MB 0 30, bytestream 2651 [h264 @ 0x5649d5bea980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d5bea980] error while decoding MB 0 33, bytestream 5226 [h264 @ 0x5649d51d1b80] cabac decode of qscale diff failed at 74 30 [h264 @ 0x5649d51d1b80] error while decoding MB 74 30, bytestream 1426 [h264 @ 0x5649d5bea980] left block unavailable for requested intr", "last_exit_at": "2026-07-01T22:01:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 618.75, "last_started_at": "2026-07-01T21:51:31+09:00", "pid": 25322, "process": {"exists": false, "pid": 25322}, "restart_count": 185, "restart_count_per_hour": 4, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 186}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.293, "sample_count": 2686, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132070, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:01:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T22:01:49+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T22:01:49+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T22:01:49.651960+09:00", "mtime_epoch": 1782910909.65196, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113251}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:01:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 924.613, "name": "celery", "pid": 237, "rss_mb": 919.199, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.368, "pushed": 3.368, "read": 3.335}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.569, "count": 101, "max_ms": 2.046, "p95_ms": 2.017}, "gstreamer_push": {"avg_ms": 1.593, "count": 101, "max_ms": 4.026, "p95_ms": 3.701}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.567, "count": 101, "max_ms": 78.339, "p95_ms": 76.489}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.013, "count": 101, "max_ms": 82.324, "p95_ms": 78.645}, "yolo11_640_fire_11class": {"avg_ms": 73.903, "count": 101, "max_ms": 80.891, "p95_ms": 78.618}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.341, "count": 101, "max_ms": 86.428, "p95_ms": 85.695}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.104, "count": 101, "max_ms": 99.258, "p95_ms": 96.045}, "read_interval": {"avg_ms": 299.7, "count": 100, "max_ms": 10133.369, "p95_ms": 115.487}, "triton_wait": {"avg_ms": 81.588, "count": 101, "max_ms": 86.44, "p95_ms": 85.707}, "visualize": {"avg_ms": 3.727, "count": 101, "max_ms": 4.936, "p95_ms": 3.979}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1100}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4233.949, "limit_mb": 8796093022207.996}, "detail": {"pid": 25733, "restart_count": 186, "start_count": 187}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 161, "h264_decode_error_count": 191, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:01:49+09:00", "last_error_summary": "[h264 @ 0x5649d50bb240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d50bb240] error while decoding MB 0 30, bytestream 2651 [h264 @ 0x5649d5bea980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5649d5bea980] error while decoding MB 0 33, bytestream 5226 [h264 @ 0x5649d51d1b80] cabac decode of qscale diff failed at 74 30 [h264 @ 0x5649d51d1b80] error while decoding MB 74 30, bytestream 1426 [h264 @ 0x5649d5bea980] left block unavailable for requested intr", "last_exit_at": "2026-07-01T22:01:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:01:51+09:00", "pid": 25733, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.676, "name": "ffmpeg", "pid": 25733, "rss_mb": 149.676, "state": "S (sleeping)", "threads": 106}, "restart_count": 186, "restart_count_per_hour": 5, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 187}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.224, "sample_count": 2652, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132070, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:01:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.059, "last_frame_pushed_at": "2026-07-01T22:01:49+09:00", "last_frame_read_age_sec": 4.155, "last_frame_read_at": "2026-07-01T22:01:49+09:00", "mjpeg": {"age_sec": 4.048, "exists": true, "mtime": "2026-07-01T22:01:49.755961+09:00", "mtime_epoch": 1782910909.7559612, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114712}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:01:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 924.613, "name": "celery", "pid": 237, "rss_mb": 919.199, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.136, "pushed": 2.136, "read": 2.102}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.564, "count": 64, "max_ms": 2.046, "p95_ms": 2.017}, "gstreamer_push": {"avg_ms": 1.606, "count": 64, "max_ms": 4.026, "p95_ms": 3.726}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.279, "count": 64, "max_ms": 77.809, "p95_ms": 74.526}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.089, "count": 64, "max_ms": 82.324, "p95_ms": 79.845}, "yolo11_640_fire_11class": {"avg_ms": 73.776, "count": 64, "max_ms": 80.891, "p95_ms": 79.442}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.687, "count": 64, "max_ms": 86.428, "p95_ms": 85.277}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.56, "count": 64, "max_ms": 97.315, "p95_ms": 95.841}, "read_interval": {"avg_ms": 411.192, "count": 63, "max_ms": 10133.369, "p95_ms": 115.487}, "triton_wait": {"avg_ms": 81.069, "count": 64, "max_ms": 86.44, "p95_ms": 85.289}, "visualize": {"avg_ms": 3.713, "count": 64, "max_ms": 4.198, "p95_ms": 3.972}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4124.746, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56027d9866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027d9866c0] error while decoding MB 0 19, bytestream 8865 [h264 @ 0x56027e4c41c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e4c41c0] error while decoding MB 0 32, bytestream 2929 [h264 @ 0x56027e3264c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e3264c0] error while decoding MB 0 32, bytestream 23155 [in#0/rtsp @ 0x56027d96be00] Error during demuxing: Con", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 162, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:07:21+09:00", "last_error_summary": "[h264 @ 0x56027d9866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027d9866c0] error while decoding MB 0 19, bytestream 8865 [h264 @ 0x56027e4c41c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e4c41c0] error while decoding MB 0 32, bytestream 2929 [h264 @ 0x56027e3264c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e3264c0] error while decoding MB 0 32, bytestream 23155 [in#0/rtsp @ 0x56027d96be00] Error during demuxing: Con", "last_exit_at": "2026-07-01T22:01:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 329.384, "last_started_at": "2026-07-01T22:01:51+09:00", "pid": 25733, "process": {"exists": false, "pid": 25733}, "restart_count": 186, "restart_count_per_hour": 5, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 187}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.034, "sample_count": 2558, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 134880, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:07:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-01T22:07:21+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-01T22:07:21+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T22:07:21.128518+09:00", "mtime_epoch": 1782911241.1285176, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 138026}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:07:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 926.219, "name": "celery", "pid": 237, "rss_mb": 923.582, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.337, "pushed": 3.337, "read": 3.337}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.557, "count": 100, "max_ms": 2.604, "p95_ms": 1.628}, "gstreamer_push": {"avg_ms": 1.646, "count": 100, "max_ms": 4.491, "p95_ms": 3.844}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.539, "count": 100, "max_ms": 82.8, "p95_ms": 78.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.517, "count": 100, "max_ms": 83.31, "p95_ms": 80.386}, "yolo11_640_fire_11class": {"avg_ms": 73.896, "count": 100, "max_ms": 82.215, "p95_ms": 78.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.033, "count": 100, "max_ms": 92.476, "p95_ms": 88.115}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.673, "count": 100, "max_ms": 102.842, "p95_ms": 98.468}, "read_interval": {"avg_ms": 299.959, "count": 100, "max_ms": 10124.238, "p95_ms": 116.577}, "triton_wait": {"avg_ms": 83.061, "count": 100, "max_ms": 92.488, "p95_ms": 88.127}, "visualize": {"avg_ms": 3.746, "count": 100, "max_ms": 4.845, "p95_ms": 4.228}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4123.996, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:07:21+09:00", "last_error_summary": "[h264 @ 0x56027d9866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027d9866c0] error while decoding MB 0 19, bytestream 8865 [h264 @ 0x56027e4c41c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e4c41c0] error while decoding MB 0 32, bytestream 2929 [h264 @ 0x56027e3264c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e3264c0] error while decoding MB 0 32, bytestream 23155 [in#0/rtsp @ 0x56027d96be00] Error during demuxing: Con", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 162, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:07:21+09:00", "last_error_summary": "[h264 @ 0x56027d9866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027d9866c0] error while decoding MB 0 19, bytestream 8865 [h264 @ 0x56027e4c41c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e4c41c0] error while decoding MB 0 32, bytestream 2929 [h264 @ 0x56027e3264c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e3264c0] error while decoding MB 0 32, bytestream 23155 [in#0/rtsp @ 0x56027d96be00] Error during demuxing: Con", "last_exit_at": "2026-07-01T22:07:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 329.436, "last_started_at": "2026-07-01T22:01:51+09:00", "pid": 25733, "process": {"exists": false, "pid": 25733}, "restart_count": 186, "restart_count_per_hour": 5, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 187}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.034, "sample_count": 2559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 134881, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:07:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T22:07:21+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T22:07:21+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-01T22:07:21.128518+09:00", "mtime_epoch": 1782911241.1285176, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 138026}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:07:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 926.219, "name": "celery", "pid": 237, "rss_mb": 923.582, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.344, "pushed": 3.344, "read": 3.31}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.562, "count": 100, "max_ms": 2.604, "p95_ms": 1.937}, "gstreamer_push": {"avg_ms": 1.648, "count": 100, "max_ms": 4.491, "p95_ms": 3.844}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.456, "count": 100, "max_ms": 82.8, "p95_ms": 78.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.57, "count": 100, "max_ms": 83.31, "p95_ms": 80.386}, "yolo11_640_fire_11class": {"avg_ms": 73.934, "count": 100, "max_ms": 82.215, "p95_ms": 78.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.075, "count": 100, "max_ms": 92.476, "p95_ms": 88.115}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.723, "count": 100, "max_ms": 102.842, "p95_ms": 98.468}, "read_interval": {"avg_ms": 301.922, "count": 99, "max_ms": 10124.238, "p95_ms": 118.227}, "triton_wait": {"avg_ms": 83.103, "count": 100, "max_ms": 92.488, "p95_ms": 88.127}, "visualize": {"avg_ms": 3.746, "count": 100, "max_ms": 4.845, "p95_ms": 4.228}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4124.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 162, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:07:21+09:00", "last_error_summary": "[h264 @ 0x56027d9866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027d9866c0] error while decoding MB 0 19, bytestream 8865 [h264 @ 0x56027e4c41c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e4c41c0] error while decoding MB 0 32, bytestream 2929 [h264 @ 0x56027e3264c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e3264c0] error while decoding MB 0 32, bytestream 23155 [in#0/rtsp @ 0x56027d96be00] Error during demuxing: Con", "last_exit_at": "2026-07-01T22:07:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 329.444, "last_started_at": "2026-07-01T22:01:51+09:00", "pid": 25733, "process": {"exists": false, "pid": 25733}, "restart_count": 186, "restart_count_per_hour": 5, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 187}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.034, "sample_count": 2558, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 134881, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:07:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.02, "last_frame_pushed_at": "2026-07-01T22:07:21+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-01T22:07:21+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-01T22:07:21.128518+09:00", "mtime_epoch": 1782911241.1285176, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 138026}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:07:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 926.219, "name": "celery", "pid": 237, "rss_mb": 923.582, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.343, "pushed": 3.343, "read": 3.309}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.562, "count": 100, "max_ms": 2.604, "p95_ms": 1.937}, "gstreamer_push": {"avg_ms": 1.648, "count": 100, "max_ms": 4.491, "p95_ms": 3.844}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.456, "count": 100, "max_ms": 82.8, "p95_ms": 78.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.57, "count": 100, "max_ms": 83.31, "p95_ms": 80.386}, "yolo11_640_fire_11class": {"avg_ms": 73.934, "count": 100, "max_ms": 82.215, "p95_ms": 78.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.075, "count": 100, "max_ms": 92.476, "p95_ms": 88.115}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.723, "count": 100, "max_ms": 102.842, "p95_ms": 98.468}, "read_interval": {"avg_ms": 301.922, "count": 99, "max_ms": 10124.238, "p95_ms": 118.227}, "triton_wait": {"avg_ms": 83.103, "count": 100, "max_ms": 92.488, "p95_ms": 88.127}, "visualize": {"avg_ms": 3.746, "count": 100, "max_ms": 4.845, "p95_ms": 4.228}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1124}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4235.109, "limit_mb": 8796093022207.996}, "detail": {"pid": 25840, "restart_count": 187, "start_count": 188}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 162, "h264_decode_error_count": 194, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:07:21+09:00", "last_error_summary": "[h264 @ 0x56027d9866c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027d9866c0] error while decoding MB 0 19, bytestream 8865 [h264 @ 0x56027e4c41c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e4c41c0] error while decoding MB 0 32, bytestream 2929 [h264 @ 0x56027e3264c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56027e3264c0] error while decoding MB 0 32, bytestream 23155 [in#0/rtsp @ 0x56027d96be00] Error during demuxing: Con", "last_exit_at": "2026-07-01T22:07:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:07:23+09:00", "pid": 25840, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.73, "name": "ffmpeg", "pid": 25840, "rss_mb": 144.73, "state": "S (sleeping)", "threads": 106}, "restart_count": 187, "restart_count_per_hour": 6, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 188}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.965, "sample_count": 2518, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 134881, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:07:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.537, "last_frame_pushed_at": "2026-07-01T22:07:21+09:00", "last_frame_read_age_sec": 4.634, "last_frame_read_at": "2026-07-01T22:07:21+09:00", "mjpeg": {"age_sec": 4.523, "exists": true, "mtime": "2026-07-01T22:07:21.240519+09:00", "mtime_epoch": 1782911241.2405193, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 138514}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:07:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 926.219, "name": "celery", "pid": 237, "rss_mb": 923.582, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.973, "pushed": 1.973, "read": 1.939}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.577, "count": 59, "max_ms": 2.604, "p95_ms": 2.09}, "gstreamer_push": {"avg_ms": 1.617, "count": 59, "max_ms": 4.491, "p95_ms": 3.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.072, "count": 59, "max_ms": 82.8, "p95_ms": 79.238}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.399, "count": 59, "max_ms": 83.31, "p95_ms": 81.808}, "yolo11_640_fire_11class": {"avg_ms": 73.932, "count": 59, "max_ms": 82.215, "p95_ms": 78.871}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.873, "count": 59, "max_ms": 92.476, "p95_ms": 88.716}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.71, "count": 59, "max_ms": 102.842, "p95_ms": 99.405}, "read_interval": {"avg_ms": 437.358, "count": 58, "max_ms": 10124.238, "p95_ms": 119.784}, "triton_wait": {"avg_ms": 82.911, "count": 59, "max_ms": 92.488, "p95_ms": 88.729}, "visualize": {"avg_ms": 3.805, "count": 59, "max_ms": 4.845, "p95_ms": 4.439}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2098}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4072.402, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55671c8af200] missing picture in access unit with size 18 [h264 @ 0x55671d09fe80] no frame! [in#0/rtsp @ 0x55671c8aae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55671c8aae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55671c8af840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:10:36+09:00", "last_error_summary": "[NULL @ 0x55671c8af200] missing picture in access unit with size 18 [h264 @ 0x55671d09fe80] no frame! [in#0/rtsp @ 0x55671c8aae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55671c8aae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55671c8af840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T21:58:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 715.773, "last_started_at": "2026-07-01T21:58:40+09:00", "pid": 25581, "process": {"exists": false, "pid": 25581}, "restart_count": 21, "restart_count_per_hour": 10, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.6, "sample_count": 2674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 251801, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:10:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-01T22:10:35+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-01T22:10:35+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-01T22:10:35.955375+09:00", "mtime_epoch": 1782911435.9553747, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93960}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:10:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 986.777, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 101, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.368, "pushed": 3.368, "read": 3.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.388, "count": 101, "max_ms": 4.524, "p95_ms": 3.476}, "gstreamer_push": {"avg_ms": 1.492, "count": 101, "max_ms": 3.962, "p95_ms": 3.677}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.298, "count": 101, "max_ms": 75.109, "p95_ms": 73.205}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.982, "count": 101, "max_ms": 79.055, "p95_ms": 76.649}, "yolo11_640_fire_11class": {"avg_ms": 69.425, "count": 101, "max_ms": 80.881, "p95_ms": 76.048}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.345, "count": 101, "max_ms": 87.757, "p95_ms": 84.927}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.588, "count": 101, "max_ms": 101.284, "p95_ms": 97.001}, "read_interval": {"avg_ms": 297.23, "count": 101, "max_ms": 19094.98, "p95_ms": 115.17}, "triton_wait": {"avg_ms": 80.373, "count": 101, "max_ms": 87.77, "p95_ms": 84.939}, "visualize": {"avg_ms": 3.673, "count": 101, "max_ms": 4.238, "p95_ms": 3.782}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2098}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4071.945, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:10:36+09:00", "last_error_summary": "[NULL @ 0x55671c8af200] missing picture in access unit with size 18 [h264 @ 0x55671d09fe80] no frame! [in#0/rtsp @ 0x55671c8aae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55671c8aae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55671c8af840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:10:36+09:00", "last_error_summary": "[NULL @ 0x55671c8af200] missing picture in access unit with size 18 [h264 @ 0x55671d09fe80] no frame! [in#0/rtsp @ 0x55671c8aae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55671c8aae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55671c8af840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:10:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 715.818, "last_started_at": "2026-07-01T21:58:40+09:00", "pid": 25581, "process": {"exists": false, "pid": 25581}, "restart_count": 21, "restart_count_per_hour": 10, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.6, "sample_count": 2674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 251802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:10:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T22:10:36+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-01T22:10:35+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-01T22:10:35.955375+09:00", "mtime_epoch": 1782911435.9553747, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93960}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:10:36+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 986.777, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.376, "pushed": 3.376, "read": 3.342}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.388, "count": 101, "max_ms": 4.524, "p95_ms": 3.476}, "gstreamer_push": {"avg_ms": 1.493, "count": 101, "max_ms": 3.962, "p95_ms": 3.677}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.37, "count": 101, "max_ms": 75.109, "p95_ms": 73.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.988, "count": 101, "max_ms": 79.055, "p95_ms": 76.649}, "yolo11_640_fire_11class": {"avg_ms": 69.496, "count": 101, "max_ms": 80.881, "p95_ms": 76.048}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.354, "count": 101, "max_ms": 87.757, "p95_ms": 84.927}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.599, "count": 101, "max_ms": 101.284, "p95_ms": 97.001}, "read_interval": {"avg_ms": 299.095, "count": 100, "max_ms": 19094.98, "p95_ms": 115.17}, "triton_wait": {"avg_ms": 80.382, "count": 101, "max_ms": 87.77, "p95_ms": 84.939}, "visualize": {"avg_ms": 3.673, "count": 101, "max_ms": 4.238, "p95_ms": 3.782}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2098}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4071.883, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:10:36+09:00", "last_error_summary": "[NULL @ 0x55671c8af200] missing picture in access unit with size 18 [h264 @ 0x55671d09fe80] no frame! [in#0/rtsp @ 0x55671c8aae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55671c8aae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55671c8af840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:10:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 715.826, "last_started_at": "2026-07-01T21:58:40+09:00", "pid": 25581, "process": {"exists": false, "pid": 25581}, "restart_count": 21, "restart_count_per_hour": 10, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.6, "sample_count": 2674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 251802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:10:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T22:10:36+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T22:10:35+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T22:10:35.955375+09:00", "mtime_epoch": 1782911435.9553747, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93960}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:10:36+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 986.777, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.375, "pushed": 3.375, "read": 3.341}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.388, "count": 101, "max_ms": 4.524, "p95_ms": 3.476}, "gstreamer_push": {"avg_ms": 1.493, "count": 101, "max_ms": 3.962, "p95_ms": 3.677}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.37, "count": 101, "max_ms": 75.109, "p95_ms": 73.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.988, "count": 101, "max_ms": 79.055, "p95_ms": 76.649}, "yolo11_640_fire_11class": {"avg_ms": 69.496, "count": 101, "max_ms": 80.881, "p95_ms": 76.048}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.354, "count": 101, "max_ms": 87.757, "p95_ms": 84.927}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.599, "count": 101, "max_ms": 101.284, "p95_ms": 97.001}, "read_interval": {"avg_ms": 299.095, "count": 100, "max_ms": 19094.98, "p95_ms": 115.17}, "triton_wait": {"avg_ms": 80.382, "count": 101, "max_ms": 87.77, "p95_ms": 84.939}, "visualize": {"avg_ms": 3.673, "count": 101, "max_ms": 4.238, "p95_ms": 3.782}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2098}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4092.66, "limit_mb": 8796093022207.996}, "detail": {"pid": 25947, "restart_count": 22, "start_count": 23}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:10:36+09:00", "last_error_summary": "[NULL @ 0x55671c8af200] missing picture in access unit with size 18 [h264 @ 0x55671d09fe80] no frame! [in#0/rtsp @ 0x55671c8aae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55671c8aae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55671c8af840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:10:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:10:38+09:00", "pid": 25947, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.359, "name": "ffmpeg", "pid": 25947, "rss_mb": 57.359, "state": "S (sleeping)", "threads": 1}, "restart_count": 22, "restart_count_per_hour": 11, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.627, "sample_count": 2634, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 251802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:10:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.448, "last_frame_pushed_at": "2026-07-01T22:10:36+09:00", "last_frame_read_age_sec": 4.546, "last_frame_read_at": "2026-07-01T22:10:35+09:00", "mjpeg": {"age_sec": 4.43, "exists": true, "mtime": "2026-07-01T22:10:36.067376+09:00", "mtime_epoch": 1782911436.0673764, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94532}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:10:40+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 986.777, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 60, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.001, "pushed": 2.001, "read": 2.001}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.378, "count": 60, "max_ms": 3.731, "p95_ms": 3.476}, "gstreamer_push": {"avg_ms": 1.429, "count": 60, "max_ms": 3.906, "p95_ms": 1.448}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.879, "count": 60, "max_ms": 74.478, "p95_ms": 73.328}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.59, "count": 60, "max_ms": 79.055, "p95_ms": 77.309}, "yolo11_640_fire_11class": {"avg_ms": 70.136, "count": 60, "max_ms": 80.881, "p95_ms": 76.642}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.783, "count": 60, "max_ms": 87.167, "p95_ms": 85.653}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.057, "count": 60, "max_ms": 99.571, "p95_ms": 97.903}, "read_interval": {"avg_ms": 425.915, "count": 60, "max_ms": 19094.98, "p95_ms": 115.17}, "triton_wait": {"avg_ms": 80.821, "count": 60, "max_ms": 87.179, "p95_ms": 85.665}, "visualize": {"avg_ms": 3.689, "count": 60, "max_ms": 4.238, "p95_ms": 3.832}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4121.242, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556acad396c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acad396c0] error while decoding MB 0 11, bytestream 29265 [h264 @ 0x556acb9ac180] error while decoding MB 32 11, bytestream -5 [h264 @ 0x556acae0a940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acae0a940] error while decoding MB 0 22, bytestream 6670 [h264 @ 0x556acad6b700] left block unavailable for requested intra mode [h264 @ 0x556acad6b700] error while decoding MB 0 8, bytestrea", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 163, "h264_decode_error_count": 200, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:13:34+09:00", "last_error_summary": "[h264 @ 0x556acad396c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acad396c0] error while decoding MB 0 11, bytestream 29265 [h264 @ 0x556acb9ac180] error while decoding MB 32 11, bytestream -5 [h264 @ 0x556acae0a940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acae0a940] error while decoding MB 0 22, bytestream 6670 [h264 @ 0x556acad6b700] left block unavailable for requested intra mode [h264 @ 0x556acad6b700] error while decoding MB 0 8, bytestrea", "last_exit_at": "2026-07-01T22:07:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 370.654, "last_started_at": "2026-07-01T22:07:23+09:00", "pid": 25840, "process": {"exists": false, "pid": 25840}, "restart_count": 187, "restart_count_per_hour": 5, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 188}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.911, "sample_count": 2649, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 138166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:13:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T22:13:34+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T22:13:34+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T22:13:34.354063+09:00", "mtime_epoch": 1782911614.354063, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 98898}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:13:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 930.332, "name": "celery", "pid": 237, "rss_mb": 922.48, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.304, "pushed": 3.304, "read": 3.304}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.535, "count": 99, "max_ms": 2.075, "p95_ms": 1.979}, "gstreamer_push": {"avg_ms": 1.645, "count": 99, "max_ms": 6.365, "p95_ms": 4.148}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.215, "count": 99, "max_ms": 80.192, "p95_ms": 78.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.334, "count": 99, "max_ms": 81.069, "p95_ms": 79.367}, "yolo11_640_fire_11class": {"avg_ms": 74.193, "count": 99, "max_ms": 80.233, "p95_ms": 78.384}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.899, "count": 99, "max_ms": 89.446, "p95_ms": 87.562}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.491, "count": 99, "max_ms": 99.892, "p95_ms": 98.34}, "read_interval": {"avg_ms": 302.994, "count": 99, "max_ms": 10126.865, "p95_ms": 120.642}, "triton_wait": {"avg_ms": 82.936, "count": 99, "max_ms": 89.458, "p95_ms": 87.578}, "visualize": {"avg_ms": 3.741, "count": 99, "max_ms": 5.177, "p95_ms": 4.369}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4120.641, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:13:34+09:00", "last_error_summary": "[h264 @ 0x556acad396c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acad396c0] error while decoding MB 0 11, bytestream 29265 [h264 @ 0x556acb9ac180] error while decoding MB 32 11, bytestream -5 [h264 @ 0x556acae0a940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acae0a940] error while decoding MB 0 22, bytestream 6670 [h264 @ 0x556acad6b700] left block unavailable for requested intra mode [h264 @ 0x556acad6b700] error while decoding MB 0 8, bytestrea", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 163, "h264_decode_error_count": 200, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:13:34+09:00", "last_error_summary": "[h264 @ 0x556acad396c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acad396c0] error while decoding MB 0 11, bytestream 29265 [h264 @ 0x556acb9ac180] error while decoding MB 32 11, bytestream -5 [h264 @ 0x556acae0a940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acae0a940] error while decoding MB 0 22, bytestream 6670 [h264 @ 0x556acad6b700] left block unavailable for requested intra mode [h264 @ 0x556acad6b700] error while decoding MB 0 8, bytestrea", "last_exit_at": "2026-07-01T22:13:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 370.683, "last_started_at": "2026-07-01T22:07:23+09:00", "pid": 25840, "process": {"exists": false, "pid": 25840}, "restart_count": 187, "restart_count_per_hour": 5, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 188}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.911, "sample_count": 2650, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 138167, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:13:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T22:13:34+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-01T22:13:34+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T22:13:34.354063+09:00", "mtime_epoch": 1782911614.354063, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 98898}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:13:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 930.332, "name": "celery", "pid": 237, "rss_mb": 922.48, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.334, "pushed": 3.334, "read": 3.3}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.533, "count": 99, "max_ms": 2.075, "p95_ms": 1.979}, "gstreamer_push": {"avg_ms": 1.642, "count": 100, "max_ms": 6.365, "p95_ms": 3.635}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.029, "count": 99, "max_ms": 80.192, "p95_ms": 78.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.287, "count": 99, "max_ms": 81.069, "p95_ms": 79.367}, "yolo11_640_fire_11class": {"avg_ms": 74.137, "count": 99, "max_ms": 80.233, "p95_ms": 78.384}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.827, "count": 99, "max_ms": 89.446, "p95_ms": 87.562}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.404, "count": 100, "max_ms": 99.892, "p95_ms": 98.307}, "read_interval": {"avg_ms": 302.994, "count": 99, "max_ms": 10126.865, "p95_ms": 120.642}, "triton_wait": {"avg_ms": 82.865, "count": 99, "max_ms": 89.458, "p95_ms": 87.578}, "visualize": {"avg_ms": 3.729, "count": 99, "max_ms": 5.177, "p95_ms": 4.369}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4120.539, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 163, "h264_decode_error_count": 200, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:13:34+09:00", "last_error_summary": "[h264 @ 0x556acad396c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acad396c0] error while decoding MB 0 11, bytestream 29265 [h264 @ 0x556acb9ac180] error while decoding MB 32 11, bytestream -5 [h264 @ 0x556acae0a940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acae0a940] error while decoding MB 0 22, bytestream 6670 [h264 @ 0x556acad6b700] left block unavailable for requested intra mode [h264 @ 0x556acad6b700] error while decoding MB 0 8, bytestrea", "last_exit_at": "2026-07-01T22:13:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 370.691, "last_started_at": "2026-07-01T22:07:23+09:00", "pid": 25840, "process": {"exists": false, "pid": 25840}, "restart_count": 187, "restart_count_per_hour": 5, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 188}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.911, "sample_count": 2649, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 138167, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:13:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T22:13:34+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-01T22:13:34+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-01T22:13:34.354063+09:00", "mtime_epoch": 1782911614.354063, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 98898}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:13:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 930.332, "name": "celery", "pid": 237, "rss_mb": 922.48, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.533, "count": 99, "max_ms": 2.075, "p95_ms": 1.979}, "gstreamer_push": {"avg_ms": 1.645, "count": 99, "max_ms": 6.365, "p95_ms": 4.148}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.029, "count": 99, "max_ms": 80.192, "p95_ms": 78.051}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.287, "count": 99, "max_ms": 81.069, "p95_ms": 79.367}, "yolo11_640_fire_11class": {"avg_ms": 74.137, "count": 99, "max_ms": 80.233, "p95_ms": 78.384}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.827, "count": 99, "max_ms": 89.446, "p95_ms": 87.562}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.399, "count": 99, "max_ms": 99.892, "p95_ms": 98.34}, "read_interval": {"avg_ms": 302.994, "count": 99, "max_ms": 10126.865, "p95_ms": 120.642}, "triton_wait": {"avg_ms": 82.865, "count": 99, "max_ms": 89.458, "p95_ms": 87.578}, "visualize": {"avg_ms": 3.729, "count": 99, "max_ms": 5.177, "p95_ms": 4.369}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1151}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4234.141, "limit_mb": 8796093022207.996}, "detail": {"pid": 26099, "restart_count": 188, "start_count": 189}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 163, "h264_decode_error_count": 200, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:13:34+09:00", "last_error_summary": "[h264 @ 0x556acad396c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acad396c0] error while decoding MB 0 11, bytestream 29265 [h264 @ 0x556acb9ac180] error while decoding MB 32 11, bytestream -5 [h264 @ 0x556acae0a940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556acae0a940] error while decoding MB 0 22, bytestream 6670 [h264 @ 0x556acad6b700] left block unavailable for requested intra mode [h264 @ 0x556acad6b700] error while decoding MB 0 8, bytestrea", "last_exit_at": "2026-07-01T22:13:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:13:37+09:00", "pid": 26099, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.891, "name": "ffmpeg", "pid": 26099, "rss_mb": 138.891, "state": "S (sleeping)", "threads": 106}, "restart_count": 188, "restart_count_per_hour": 6, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 189}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.916, "sample_count": 2604, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 138167, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:13:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.984, "last_frame_pushed_at": "2026-07-01T22:13:34+09:00", "last_frame_read_age_sec": 5.071, "last_frame_read_at": "2026-07-01T22:13:34+09:00", "mjpeg": {"age_sec": 4.969, "exists": true, "mtime": "2026-07-01T22:13:34.454064+09:00", "mtime_epoch": 1782911614.4540644, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 98759}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:13:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 930.332, "name": "celery", "pid": 237, "rss_mb": 922.48, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.838, "pushed": 1.838, "read": 1.805}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.504, "count": 55, "max_ms": 2.05, "p95_ms": 1.61}, "gstreamer_push": {"avg_ms": 1.558, "count": 55, "max_ms": 6.184, "p95_ms": 3.635}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.973, "count": 55, "max_ms": 80.192, "p95_ms": 77.879}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.678, "count": 55, "max_ms": 80.2, "p95_ms": 78.766}, "yolo11_640_fire_11class": {"avg_ms": 73.891, "count": 55, "max_ms": 80.183, "p95_ms": 78.064}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.494, "count": 55, "max_ms": 87.25, "p95_ms": 86.847}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.843, "count": 55, "max_ms": 99.207, "p95_ms": 97.529}, "read_interval": {"avg_ms": 461.95, "count": 54, "max_ms": 10126.865, "p95_ms": 120.882}, "triton_wait": {"avg_ms": 81.552, "count": 55, "max_ms": 87.262, "p95_ms": 86.86}, "visualize": {"avg_ms": 3.641, "count": 55, "max_ms": 4.369, "p95_ms": 4.084}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2118}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4071.785, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559beb0a9a00] error while decoding MB 98 12, bytestream -5 [in#0/rtsp @ 0x559bea7d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559bea7d8e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:15:22+09:00", "last_error_summary": "[h264 @ 0x559beb0a9a00] error while decoding MB 98 12, bytestream -5 [in#0/rtsp @ 0x559bea7d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559bea7d8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:10:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 284.354, "last_started_at": "2026-07-01T22:10:38+09:00", "pid": 25947, "process": {"exists": false, "pid": 25947}, "restart_count": 22, "restart_count_per_hour": 9, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.565, "sample_count": 2390, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 254180, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:15:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-01T22:15:22+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-01T22:15:22+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T22:15:22.791722+09:00", "mtime_epoch": 1782911722.7917216, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98149}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:15:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 987.938, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.684, "pushed": 0.684, "read": 0.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.354, "count": 12, "max_ms": 3.431, "p95_ms": 3.431}, "gstreamer_push": {"avg_ms": 1.327, "count": 12, "max_ms": 1.373, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.469, "count": 12, "max_ms": 73.243, "p95_ms": 73.243}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.186, "count": 12, "max_ms": 80.573, "p95_ms": 80.573}, "yolo11_640_fire_11class": {"avg_ms": 71.64, "count": 12, "max_ms": 78.749, "p95_ms": 78.749}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.989, "count": 12, "max_ms": 84.242, "p95_ms": 84.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.369, "count": 12, "max_ms": 96.13, "p95_ms": 96.13}, "read_interval": {"avg_ms": 2516.453, "count": 13, "max_ms": 16305.459, "p95_ms": 16305.459}, "triton_wait": {"avg_ms": 78.278, "count": 12, "max_ms": 84.253, "p95_ms": 84.253}, "visualize": {"avg_ms": 3.677, "count": 12, "max_ms": 4.028, "p95_ms": 4.028}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.554, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2118}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4071.305, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:15:22+09:00", "last_error_summary": "[h264 @ 0x559beb0a9a00] error while decoding MB 98 12, bytestream -5 [in#0/rtsp @ 0x559bea7d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559bea7d8e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:15:22+09:00", "last_error_summary": "[h264 @ 0x559beb0a9a00] error while decoding MB 98 12, bytestream -5 [in#0/rtsp @ 0x559bea7d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559bea7d8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:15:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 284.39, "last_started_at": "2026-07-01T22:10:38+09:00", "pid": 25947, "process": {"exists": false, "pid": 25947}, "restart_count": 22, "restart_count_per_hour": 9, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.561, "sample_count": 2391, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 254181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:15:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T22:15:22+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-01T22:15:22+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-01T22:15:22.791722+09:00", "mtime_epoch": 1782911722.7917216, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98149}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:15:22+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 987.938, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.739, "pushed": 0.739, "read": 0.739}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.347, "count": 13, "max_ms": 3.431, "p95_ms": 3.431}, "gstreamer_push": {"avg_ms": 1.331, "count": 13, "max_ms": 1.375, "p95_ms": 1.375}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.777, "count": 13, "max_ms": 73.243, "p95_ms": 73.243}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.257, "count": 13, "max_ms": 80.573, "p95_ms": 80.573}, "yolo11_640_fire_11class": {"avg_ms": 72.356, "count": 13, "max_ms": 80.946, "p95_ms": 80.946}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.216, "count": 13, "max_ms": 84.242, "p95_ms": 84.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.594, "count": 13, "max_ms": 96.13, "p95_ms": 96.13}, "read_interval": {"avg_ms": 2516.453, "count": 13, "max_ms": 16305.459, "p95_ms": 16305.459}, "triton_wait": {"avg_ms": 78.484, "count": 13, "max_ms": 84.253, "p95_ms": 84.253}, "visualize": {"avg_ms": 3.701, "count": 13, "max_ms": 4.028, "p95_ms": 4.028}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.589, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2118}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4071.309, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:15:22+09:00", "last_error_summary": "[h264 @ 0x559beb0a9a00] error while decoding MB 98 12, bytestream -5 [in#0/rtsp @ 0x559bea7d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559bea7d8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:15:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 284.397, "last_started_at": "2026-07-01T22:10:38+09:00", "pid": 25947, "process": {"exists": false, "pid": 25947}, "restart_count": 22, "restart_count_per_hour": 9, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.561, "sample_count": 2391, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 254181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:15:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T22:15:22+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-01T22:15:22+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-01T22:15:22.791722+09:00", "mtime_epoch": 1782911722.7917216, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98149}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:15:22+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 987.938, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.739, "pushed": 0.739, "read": 0.739}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.347, "count": 13, "max_ms": 3.431, "p95_ms": 3.431}, "gstreamer_push": {"avg_ms": 1.331, "count": 13, "max_ms": 1.375, "p95_ms": 1.375}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.777, "count": 13, "max_ms": 73.243, "p95_ms": 73.243}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.257, "count": 13, "max_ms": 80.573, "p95_ms": 80.573}, "yolo11_640_fire_11class": {"avg_ms": 72.356, "count": 13, "max_ms": 80.946, "p95_ms": 80.946}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.216, "count": 13, "max_ms": 84.242, "p95_ms": 84.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.594, "count": 13, "max_ms": 96.13, "p95_ms": 96.13}, "read_interval": {"avg_ms": 2516.453, "count": 13, "max_ms": 16305.459, "p95_ms": 16305.459}, "triton_wait": {"avg_ms": 78.484, "count": 13, "max_ms": 84.253, "p95_ms": 84.253}, "visualize": {"avg_ms": 3.701, "count": 13, "max_ms": 4.028, "p95_ms": 4.028}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.597, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2118}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4251.18, "limit_mb": 8796093022207.996}, "detail": {"pid": 26206, "restart_count": 23, "start_count": 24}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:15:22+09:00", "last_error_summary": "[h264 @ 0x559beb0a9a00] error while decoding MB 98 12, bytestream -5 [in#0/rtsp @ 0x559bea7d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559bea7d8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:15:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:15:25+09:00", "pid": 26206, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.262, "name": "ffmpeg", "pid": 26206, "rss_mb": 209.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 23, "restart_count_per_hour": 10, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.561, "sample_count": 2391, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 254181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:15:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.374, "last_frame_pushed_at": "2026-07-01T22:15:22+09:00", "last_frame_read_age_sec": 4.47, "last_frame_read_at": "2026-07-01T22:15:22+09:00", "mjpeg": {"age_sec": 4.351, "exists": true, "mtime": "2026-07-01T22:15:22.903723+09:00", "mtime_epoch": 1782911722.9037232, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98437}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:15:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 987.938, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.592, "pushed": 0.592, "read": 0.592}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.347, "count": 13, "max_ms": 3.431, "p95_ms": 3.431}, "gstreamer_push": {"avg_ms": 1.331, "count": 13, "max_ms": 1.375, "p95_ms": 1.375}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.777, "count": 13, "max_ms": 73.243, "p95_ms": 73.243}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.257, "count": 13, "max_ms": 80.573, "p95_ms": 80.573}, "yolo11_640_fire_11class": {"avg_ms": 72.356, "count": 13, "max_ms": 80.946, "p95_ms": 80.946}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.216, "count": 13, "max_ms": 84.242, "p95_ms": 84.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.594, "count": 13, "max_ms": 96.13, "p95_ms": 96.13}, "read_interval": {"avg_ms": 2516.453, "count": 13, "max_ms": 16305.459, "p95_ms": 16305.459}, "triton_wait": {"avg_ms": 78.484, "count": 13, "max_ms": 84.253, "p95_ms": 84.253}, "visualize": {"avg_ms": 3.701, "count": 13, "max_ms": 4.028, "p95_ms": 4.028}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.958, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4092.062, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55b4680be200] missing picture in access unit with size 18 [h264 @ 0x55b468d8e140] no frame! [in#0/rtsp @ 0x55b4680b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4680b9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b4680be840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:19:00+09:00", "last_error_summary": "[NULL @ 0x55b4680be200] missing picture in access unit with size 18 [h264 @ 0x55b468d8e140] no frame! [in#0/rtsp @ 0x55b4680b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4680b9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b4680be840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:15:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 215.292, "last_started_at": "2026-07-01T22:15:25+09:00", "pid": 26206, "process": {"exists": false, "pid": 26206}, "restart_count": 23, "restart_count_per_hour": 9, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.478, "sample_count": 2169, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255888, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:19:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T22:19:00+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-01T22:19:00+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T22:19:00.483098+09:00", "mtime_epoch": 1782911940.4830978, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97684}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:19:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 988.84, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 44, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.467, "pushed": 1.467, "read": 1.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.331, "count": 44, "max_ms": 3.514, "p95_ms": 3.444}, "gstreamer_push": {"avg_ms": 1.502, "count": 44, "max_ms": 3.672, "p95_ms": 3.658}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.567, "count": 44, "max_ms": 75.707, "p95_ms": 75.192}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.786, "count": 44, "max_ms": 80.991, "p95_ms": 76.844}, "yolo11_640_fire_11class": {"avg_ms": 72.31, "count": 44, "max_ms": 75.991, "p95_ms": 75.349}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.223, "count": 44, "max_ms": 85.51, "p95_ms": 85.062}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.674, "count": 44, "max_ms": 98.063, "p95_ms": 97.633}, "read_interval": {"avg_ms": 681.934, "count": 44, "max_ms": 15988.281, "p95_ms": 119.704}, "triton_wait": {"avg_ms": 81.259, "count": 44, "max_ms": 85.524, "p95_ms": 85.074}, "visualize": {"avg_ms": 3.725, "count": 44, "max_ms": 4.204, "p95_ms": 3.916}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4086.293, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:19:00+09:00", "last_error_summary": "[NULL @ 0x55b4680be200] missing picture in access unit with size 18 [h264 @ 0x55b468d8e140] no frame! [in#0/rtsp @ 0x55b4680b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4680b9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b4680be840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:19:00+09:00", "last_error_summary": "[NULL @ 0x55b4680be200] missing picture in access unit with size 18 [h264 @ 0x55b468d8e140] no frame! [in#0/rtsp @ 0x55b4680b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4680b9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b4680be840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:19:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 215.336, "last_started_at": "2026-07-01T22:15:25+09:00", "pid": 26206, "process": {"exists": false, "pid": 26206}, "restart_count": 23, "restart_count_per_hour": 9, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.478, "sample_count": 2170, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255889, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:19:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T22:19:00+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-01T22:19:00+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T22:19:00.483098+09:00", "mtime_epoch": 1782911940.4830978, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97684}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:19:00+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 983.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.47, "pushed": 1.47, "read": 1.437}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.332, "count": 44, "max_ms": 3.514, "p95_ms": 3.444}, "gstreamer_push": {"avg_ms": 1.556, "count": 44, "max_ms": 3.743, "p95_ms": 3.664}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.834, "count": 44, "max_ms": 75.707, "p95_ms": 75.192}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.0, "count": 44, "max_ms": 80.991, "p95_ms": 76.844}, "yolo11_640_fire_11class": {"avg_ms": 72.53, "count": 44, "max_ms": 75.991, "p95_ms": 75.349}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.599, "count": 44, "max_ms": 85.51, "p95_ms": 85.062}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.097, "count": 44, "max_ms": 98.063, "p95_ms": 97.633}, "read_interval": {"avg_ms": 695.585, "count": 43, "max_ms": 15988.281, "p95_ms": 119.704}, "triton_wait": {"avg_ms": 81.635, "count": 44, "max_ms": 85.524, "p95_ms": 85.074}, "visualize": {"avg_ms": 3.741, "count": 44, "max_ms": 4.392, "p95_ms": 3.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4086.43, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:19:00+09:00", "last_error_summary": "[NULL @ 0x55b4680be200] missing picture in access unit with size 18 [h264 @ 0x55b468d8e140] no frame! [in#0/rtsp @ 0x55b4680b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4680b9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b4680be840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:19:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 215.343, "last_started_at": "2026-07-01T22:15:25+09:00", "pid": 26206, "process": {"exists": false, "pid": 26206}, "restart_count": 23, "restart_count_per_hour": 9, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.478, "sample_count": 2170, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255889, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:19:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T22:19:00+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-01T22:19:00+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-01T22:19:00.483098+09:00", "mtime_epoch": 1782911940.4830978, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97684}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:19:00+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 983.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.47, "pushed": 1.47, "read": 1.437}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.332, "count": 44, "max_ms": 3.514, "p95_ms": 3.444}, "gstreamer_push": {"avg_ms": 1.556, "count": 44, "max_ms": 3.743, "p95_ms": 3.664}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.834, "count": 44, "max_ms": 75.707, "p95_ms": 75.192}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.0, "count": 44, "max_ms": 80.991, "p95_ms": 76.844}, "yolo11_640_fire_11class": {"avg_ms": 72.53, "count": 44, "max_ms": 75.991, "p95_ms": 75.349}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.599, "count": 44, "max_ms": 85.51, "p95_ms": 85.062}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.097, "count": 44, "max_ms": 98.063, "p95_ms": 97.633}, "read_interval": {"avg_ms": 695.585, "count": 43, "max_ms": 15988.281, "p95_ms": 119.704}, "triton_wait": {"avg_ms": 81.635, "count": 44, "max_ms": 85.524, "p95_ms": 85.074}, "visualize": {"avg_ms": 3.741, "count": 44, "max_ms": 4.392, "p95_ms": 3.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2132}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4265.059, "limit_mb": 8796093022207.996}, "detail": {"pid": 26358, "restart_count": 24, "start_count": 25}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:19:00+09:00", "last_error_summary": "[NULL @ 0x55b4680be200] missing picture in access unit with size 18 [h264 @ 0x55b468d8e140] no frame! [in#0/rtsp @ 0x55b4680b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4680b9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b4680be840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:19:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:19:03+09:00", "pid": 26358, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.156, "name": "ffmpeg", "pid": 26358, "rss_mb": 209.156, "state": "S (sleeping)", "threads": 106}, "restart_count": 24, "restart_count_per_hour": 10, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.49, "sample_count": 2128, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 255889, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:19:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.563, "last_frame_pushed_at": "2026-07-01T22:19:00+09:00", "last_frame_read_age_sec": 4.665, "last_frame_read_at": "2026-07-01T22:19:00+09:00", "mjpeg": {"age_sec": 4.549, "exists": true, "mtime": "2026-07-01T22:19:00.599101+09:00", "mtime_epoch": 1782911940.5991008, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98557}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:19:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 983.426, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.595, "pushed": 0.595, "read": 0.595}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.356, "count": 13, "max_ms": 3.44, "p95_ms": 3.44}, "gstreamer_push": {"avg_ms": 1.523, "count": 13, "max_ms": 3.743, "p95_ms": 3.743}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.647, "count": 13, "max_ms": 75.707, "p95_ms": 75.707}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.738, "count": 13, "max_ms": 76.844, "p95_ms": 76.844}, "yolo11_640_fire_11class": {"avg_ms": 72.453, "count": 13, "max_ms": 75.066, "p95_ms": 75.066}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.514, "count": 13, "max_ms": 85.51, "p95_ms": 85.51}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.951, "count": 13, "max_ms": 97.703, "p95_ms": 97.703}, "read_interval": {"avg_ms": 2045.471, "count": 13, "max_ms": 15988.281, "p95_ms": 15988.281}, "triton_wait": {"avg_ms": 81.525, "count": 13, "max_ms": 85.524, "p95_ms": 85.524}, "visualize": {"avg_ms": 3.76, "count": 13, "max_ms": 4.392, "p95_ms": 4.392}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.862, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1178}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4128.875, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 15, bytestream 1326 [h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 17, bytestream 9290 [h264 @ 0x5628f14389c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f14389c0] error while decoding MB 0 26, bytestream 10026 [h264 @ 0x5628f15076c0] error while decoding MB 58 4, b", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 164, "h264_decode_error_count": 204, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:19:26+09:00", "last_error_summary": "[h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 15, bytestream 1326 [h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 17, bytestream 9290 [h264 @ 0x5628f14389c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f14389c0] error while decoding MB 0 26, bytestream 10026 [h264 @ 0x5628f15076c0] error while decoding MB 58 4, b", "last_exit_at": "2026-07-01T22:13:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 349.417, "last_started_at": "2026-07-01T22:13:37+09:00", "pid": 26099, "process": {"exists": false, "pid": 26099}, "restart_count": 188, "restart_count_per_hour": 6, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 189}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.287, "sample_count": 2778, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 141381, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:19:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-01T22:19:26+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-01T22:19:26+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-01T22:19:26.783764+09:00", "mtime_epoch": 1782911966.783764, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 101495}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:19:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 931.105, "name": "celery", "pid": 237, "rss_mb": 923.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 133, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.401, "pushed": 4.401, "read": 4.434}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.527, "count": 132, "max_ms": 1.895, "p95_ms": 1.595}, "gstreamer_push": {"avg_ms": 1.55, "count": 132, "max_ms": 5.823, "p95_ms": 3.694}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.147, "count": 132, "max_ms": 75.748, "p95_ms": 66.78}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.964, "count": 132, "max_ms": 77.48, "p95_ms": 73.977}, "yolo11_640_fire_11class": {"avg_ms": 50.506, "count": 132, "max_ms": 75.281, "p95_ms": 73.083}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.97, "count": 132, "max_ms": 84.366, "p95_ms": 82.682}}, "truncated_count": 0}, "processing_total": {"avg_ms": 62.641, "count": 132, "max_ms": 94.714, "p95_ms": 93.045}, "read_interval": {"avg_ms": 225.645, "count": 133, "max_ms": 19466.219, "p95_ms": 110.906}, "triton_wait": {"avg_ms": 52.412, "count": 132, "max_ms": 84.379, "p95_ms": 82.696}, "visualize": {"avg_ms": 3.683, "count": 132, "max_ms": 4.462, "p95_ms": 3.908}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1178}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4131.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:19:26+09:00", "last_error_summary": "[h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 15, bytestream 1326 [h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 17, bytestream 9290 [h264 @ 0x5628f14389c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f14389c0] error while decoding MB 0 26, bytestream 10026 [h264 @ 0x5628f15076c0] error while decoding MB 58 4, b", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 164, "h264_decode_error_count": 204, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:19:26+09:00", "last_error_summary": "[h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 15, bytestream 1326 [h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 17, bytestream 9290 [h264 @ 0x5628f14389c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f14389c0] error while decoding MB 0 26, bytestream 10026 [h264 @ 0x5628f15076c0] error while decoding MB 58 4, b", "last_exit_at": "2026-07-01T22:19:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 349.44, "last_started_at": "2026-07-01T22:13:37+09:00", "pid": 26099, "process": {"exists": false, "pid": 26099}, "restart_count": 188, "restart_count_per_hour": 6, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 189}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.287, "sample_count": 2779, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 141382, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:19:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T22:19:26+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-01T22:19:26+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-01T22:19:26.783764+09:00", "mtime_epoch": 1782911966.783764, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 101495}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:19:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 931.105, "name": "celery", "pid": 237, "rss_mb": 926.664, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 132, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.439, "pushed": 4.439, "read": 4.406}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.531, "count": 133, "max_ms": 1.933, "p95_ms": 1.649}, "gstreamer_push": {"avg_ms": 1.568, "count": 133, "max_ms": 5.823, "p95_ms": 3.725}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.176, "count": 133, "max_ms": 75.748, "p95_ms": 66.78}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.011, "count": 133, "max_ms": 77.48, "p95_ms": 73.977}, "yolo11_640_fire_11class": {"avg_ms": 50.552, "count": 133, "max_ms": 75.281, "p95_ms": 73.083}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.013, "count": 133, "max_ms": 84.366, "p95_ms": 82.682}}, "truncated_count": 0}, "processing_total": {"avg_ms": 62.721, "count": 133, "max_ms": 94.714, "p95_ms": 93.045}, "read_interval": {"avg_ms": 226.782, "count": 132, "max_ms": 19466.219, "p95_ms": 110.906}, "triton_wait": {"avg_ms": 52.452, "count": 133, "max_ms": 84.379, "p95_ms": 82.696}, "visualize": {"avg_ms": 3.718, "count": 133, "max_ms": 8.447, "p95_ms": 3.949}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1178}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4131.215, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 164, "h264_decode_error_count": 204, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:19:26+09:00", "last_error_summary": "[h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 15, bytestream 1326 [h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 17, bytestream 9290 [h264 @ 0x5628f14389c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f14389c0] error while decoding MB 0 26, bytestream 10026 [h264 @ 0x5628f15076c0] error while decoding MB 58 4, b", "last_exit_at": "2026-07-01T22:19:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 349.448, "last_started_at": "2026-07-01T22:13:37+09:00", "pid": 26099, "process": {"exists": false, "pid": 26099}, "restart_count": 188, "restart_count_per_hour": 6, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 189}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.287, "sample_count": 2779, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 141382, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:19:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T22:19:26+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-01T22:19:26+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-01T22:19:26.783764+09:00", "mtime_epoch": 1782911966.783764, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 101495}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:19:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 931.105, "name": "celery", "pid": 237, "rss_mb": 926.664, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 132, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.438, "pushed": 4.438, "read": 4.405}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.531, "count": 133, "max_ms": 1.933, "p95_ms": 1.649}, "gstreamer_push": {"avg_ms": 1.568, "count": 133, "max_ms": 5.823, "p95_ms": 3.725}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.176, "count": 133, "max_ms": 75.748, "p95_ms": 66.78}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.011, "count": 133, "max_ms": 77.48, "p95_ms": 73.977}, "yolo11_640_fire_11class": {"avg_ms": 50.552, "count": 133, "max_ms": 75.281, "p95_ms": 73.083}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.013, "count": 133, "max_ms": 84.366, "p95_ms": 82.682}}, "truncated_count": 0}, "processing_total": {"avg_ms": 62.721, "count": 133, "max_ms": 94.714, "p95_ms": 93.045}, "read_interval": {"avg_ms": 226.782, "count": 132, "max_ms": 19466.219, "p95_ms": 110.906}, "triton_wait": {"avg_ms": 52.452, "count": 133, "max_ms": 84.379, "p95_ms": 82.696}, "visualize": {"avg_ms": 3.718, "count": 133, "max_ms": 8.447, "p95_ms": 3.949}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1178}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4235.863, "limit_mb": 8796093022207.996}, "detail": {"pid": 26510, "restart_count": 189, "start_count": 190}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 164, "h264_decode_error_count": 204, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:19:26+09:00", "last_error_summary": "[h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 15, bytestream 1326 [h264 @ 0x5628f0be9340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f0be9340] error while decoding MB 0 17, bytestream 9290 [h264 @ 0x5628f14389c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5628f14389c0] error while decoding MB 0 26, bytestream 10026 [h264 @ 0x5628f15076c0] error while decoding MB 58 4, b", "last_exit_at": "2026-07-01T22:19:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:19:29+09:00", "pid": 26510, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.535, "name": "ffmpeg", "pid": 26510, "rss_mb": 138.535, "state": "S (sleeping)", "threads": 106}, "restart_count": 189, "restart_count_per_hour": 7, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 190}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.287, "sample_count": 2736, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 141382, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:19:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.73, "last_frame_pushed_at": "2026-07-01T22:19:26+09:00", "last_frame_read_age_sec": 4.808, "last_frame_read_at": "2026-07-01T22:19:26+09:00", "mjpeg": {"age_sec": 4.714, "exists": true, "mtime": "2026-07-01T22:19:26.871766+09:00", "mtime_epoch": 1782911966.871766, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 102726}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:19:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 931.105, "name": "celery", "pid": 237, "rss_mb": 926.664, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 75, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.538, "pushed": 2.538, "read": 2.505}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.532, "count": 76, "max_ms": 1.933, "p95_ms": 1.649}, "gstreamer_push": {"avg_ms": 1.462, "count": 76, "max_ms": 5.823, "p95_ms": 1.467}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.123, "count": 76, "max_ms": 59.668, "p95_ms": 54.263}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.408, "count": 76, "max_ms": 66.288, "p95_ms": 57.263}, "yolo11_640_fire_11class": {"avg_ms": 48.946, "count": 76, "max_ms": 60.836, "p95_ms": 56.634}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.782, "count": 76, "max_ms": 66.807, "p95_ms": 57.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.392, "count": 76, "max_ms": 77.274, "p95_ms": 69.212}, "read_interval": {"avg_ms": 335.888, "count": 75, "max_ms": 19466.219, "p95_ms": 85.174}, "triton_wait": {"avg_ms": 50.136, "count": 76, "max_ms": 66.817, "p95_ms": 57.765}, "visualize": {"avg_ms": 3.741, "count": 76, "max_ms": 8.447, "p95_ms": 3.949}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4130.941, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5624332b01c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332b01c0] error while decoding MB 0 24, bytestream 32248 [h264 @ 0x5624332b01c0] left block unavailable for requested intra mode [h264 @ 0x5624332b01c0] error while decoding MB 0 44, bytestream 7329 [h264 @ 0x5624332eadc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332eadc0] error while decoding MB 0 9, bytestream 6602 [in#0/rtsp @ 0x562433267e00] Error during demuxing: Connection", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 165, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:23:30+09:00", "last_error_summary": "[h264 @ 0x5624332b01c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332b01c0] error while decoding MB 0 24, bytestream 32248 [h264 @ 0x5624332b01c0] left block unavailable for requested intra mode [h264 @ 0x5624332b01c0] error while decoding MB 0 44, bytestream 7329 [h264 @ 0x5624332eadc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332eadc0] error while decoding MB 0 9, bytestream 6602 [in#0/rtsp @ 0x562433267e00] Error during demuxing: Connection", "last_exit_at": "2026-07-01T22:19:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.906, "last_started_at": "2026-07-01T22:19:29+09:00", "pid": 26510, "process": {"exists": false, "pid": 26510}, "restart_count": 189, "restart_count_per_hour": 6, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 190}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.138, "sample_count": 2482, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 143395, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:23:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-01T22:23:30+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-01T22:23:30+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-01T22:23:30.437505+09:00", "mtime_epoch": 1782912210.4375045, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 100916}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:23:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 931.105, "name": "celery", "pid": 237, "rss_mb": 924.602, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.407, "pushed": 3.407, "read": 3.407}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.539, "count": 102, "max_ms": 2.043, "p95_ms": 1.579}, "gstreamer_push": {"avg_ms": 1.678, "count": 102, "max_ms": 6.24, "p95_ms": 3.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.31, "count": 102, "max_ms": 78.182, "p95_ms": 75.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.283, "count": 102, "max_ms": 81.7, "p95_ms": 78.62}, "yolo11_640_fire_11class": {"avg_ms": 72.454, "count": 102, "max_ms": 82.8, "p95_ms": 78.352}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.634, "count": 102, "max_ms": 87.297, "p95_ms": 84.719}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.198, "count": 102, "max_ms": 97.765, "p95_ms": 95.636}, "read_interval": {"avg_ms": 293.881, "count": 102, "max_ms": 19186.713, "p95_ms": 116.346}, "triton_wait": {"avg_ms": 79.722, "count": 102, "max_ms": 87.309, "p95_ms": 84.731}, "visualize": {"avg_ms": 3.736, "count": 102, "max_ms": 5.426, "p95_ms": 3.961}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4130.707, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:23:30+09:00", "last_error_summary": "[h264 @ 0x5624332b01c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332b01c0] error while decoding MB 0 24, bytestream 32248 [h264 @ 0x5624332b01c0] left block unavailable for requested intra mode [h264 @ 0x5624332b01c0] error while decoding MB 0 44, bytestream 7329 [h264 @ 0x5624332eadc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332eadc0] error while decoding MB 0 9, bytestream 6602 [in#0/rtsp @ 0x562433267e00] Error during demuxing: Connection", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 165, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:23:30+09:00", "last_error_summary": "[h264 @ 0x5624332b01c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332b01c0] error while decoding MB 0 24, bytestream 32248 [h264 @ 0x5624332b01c0] left block unavailable for requested intra mode [h264 @ 0x5624332b01c0] error while decoding MB 0 44, bytestream 7329 [h264 @ 0x5624332eadc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332eadc0] error while decoding MB 0 9, bytestream 6602 [in#0/rtsp @ 0x562433267e00] Error during demuxing: Connection", "last_exit_at": "2026-07-01T22:23:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.93, "last_started_at": "2026-07-01T22:19:29+09:00", "pid": 26510, "process": {"exists": false, "pid": 26510}, "restart_count": 189, "restart_count_per_hour": 6, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 190}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.138, "sample_count": 2483, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 143396, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:23:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T22:23:30+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-01T22:23:30+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-01T22:23:30.437505+09:00", "mtime_epoch": 1782912210.4375045, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 100916}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:23:30+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 931.105, "name": "celery", "pid": 237, "rss_mb": 924.602, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.437, "pushed": 3.437, "read": 3.404}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.579, "count": 103, "max_ms": 5.624, "p95_ms": 1.615}, "gstreamer_push": {"avg_ms": 1.676, "count": 103, "max_ms": 6.24, "p95_ms": 3.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.169, "count": 103, "max_ms": 78.182, "p95_ms": 75.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.123, "count": 103, "max_ms": 81.7, "p95_ms": 78.62}, "yolo11_640_fire_11class": {"avg_ms": 72.296, "count": 103, "max_ms": 82.8, "p95_ms": 78.352}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.422, "count": 103, "max_ms": 87.297, "p95_ms": 84.719}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.034, "count": 103, "max_ms": 97.765, "p95_ms": 95.636}, "read_interval": {"avg_ms": 293.881, "count": 102, "max_ms": 19186.713, "p95_ms": 116.346}, "triton_wait": {"avg_ms": 79.509, "count": 103, "max_ms": 87.309, "p95_ms": 84.731}, "visualize": {"avg_ms": 3.747, "count": 103, "max_ms": 5.426, "p95_ms": 3.967}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4130.809, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 165, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:23:30+09:00", "last_error_summary": "[h264 @ 0x5624332b01c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332b01c0] error while decoding MB 0 24, bytestream 32248 [h264 @ 0x5624332b01c0] left block unavailable for requested intra mode [h264 @ 0x5624332b01c0] error while decoding MB 0 44, bytestream 7329 [h264 @ 0x5624332eadc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332eadc0] error while decoding MB 0 9, bytestream 6602 [in#0/rtsp @ 0x562433267e00] Error during demuxing: Connection", "last_exit_at": "2026-07-01T22:23:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 240.938, "last_started_at": "2026-07-01T22:19:29+09:00", "pid": 26510, "process": {"exists": false, "pid": 26510}, "restart_count": 189, "restart_count_per_hour": 6, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 190}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.138, "sample_count": 2483, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 143396, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:23:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T22:23:30+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-01T22:23:30+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-01T22:23:30.437505+09:00", "mtime_epoch": 1782912210.4375045, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 100916}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:23:30+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 931.105, "name": "celery", "pid": 237, "rss_mb": 924.602, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.436, "pushed": 3.436, "read": 3.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.579, "count": 103, "max_ms": 5.624, "p95_ms": 1.615}, "gstreamer_push": {"avg_ms": 1.676, "count": 103, "max_ms": 6.24, "p95_ms": 3.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.169, "count": 103, "max_ms": 78.182, "p95_ms": 75.781}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.123, "count": 103, "max_ms": 81.7, "p95_ms": 78.62}, "yolo11_640_fire_11class": {"avg_ms": 72.296, "count": 103, "max_ms": 82.8, "p95_ms": 78.352}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.422, "count": 103, "max_ms": 87.297, "p95_ms": 84.719}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.034, "count": 103, "max_ms": 97.765, "p95_ms": 95.636}, "read_interval": {"avg_ms": 293.881, "count": 102, "max_ms": 19186.713, "p95_ms": 116.346}, "triton_wait": {"avg_ms": 79.509, "count": 103, "max_ms": 87.309, "p95_ms": 84.731}, "visualize": {"avg_ms": 3.747, "count": 103, "max_ms": 5.426, "p95_ms": 3.967}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4238.516, "limit_mb": 8796093022207.996}, "detail": {"pid": 26617, "restart_count": 190, "start_count": 191}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 165, "h264_decode_error_count": 207, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:23:30+09:00", "last_error_summary": "[h264 @ 0x5624332b01c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332b01c0] error while decoding MB 0 24, bytestream 32248 [h264 @ 0x5624332b01c0] left block unavailable for requested intra mode [h264 @ 0x5624332b01c0] error while decoding MB 0 44, bytestream 7329 [h264 @ 0x5624332eadc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5624332eadc0] error while decoding MB 0 9, bytestream 6602 [in#0/rtsp @ 0x562433267e00] Error during demuxing: Connection", "last_exit_at": "2026-07-01T22:23:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:23:32+09:00", "pid": 26617, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.805, "name": "ffmpeg", "pid": 26617, "rss_mb": 141.805, "state": "S (sleeping)", "threads": 106}, "restart_count": 190, "restart_count_per_hour": 7, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 191}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.152, "sample_count": 2442, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 143396, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:23:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.221, "last_frame_pushed_at": "2026-07-01T22:23:30+09:00", "last_frame_read_age_sec": 4.297, "last_frame_read_at": "2026-07-01T22:23:30+09:00", "mjpeg": {"age_sec": 4.208, "exists": true, "mtime": "2026-07-01T22:23:30.521506+09:00", "mtime_epoch": 1782912210.5215063, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 100338}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:23:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 931.105, "name": "celery", "pid": 237, "rss_mb": 924.602, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.139, "pushed": 2.139, "read": 2.106}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.62, "count": 64, "max_ms": 5.624, "p95_ms": 2.012}, "gstreamer_push": {"avg_ms": 1.701, "count": 64, "max_ms": 3.994, "p95_ms": 3.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.789, "count": 64, "max_ms": 77.829, "p95_ms": 75.8}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.055, "count": 64, "max_ms": 78.926, "p95_ms": 77.947}, "yolo11_640_fire_11class": {"avg_ms": 70.237, "count": 64, "max_ms": 78.352, "p95_ms": 76.395}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.63, "count": 64, "max_ms": 86.265, "p95_ms": 84.546}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.363, "count": 64, "max_ms": 97.07, "p95_ms": 95.684}, "read_interval": {"avg_ms": 408.026, "count": 63, "max_ms": 19186.713, "p95_ms": 116.346}, "triton_wait": {"avg_ms": 77.727, "count": 64, "max_ms": 86.278, "p95_ms": 84.558}, "visualize": {"avg_ms": 3.779, "count": 64, "max_ms": 5.426, "p95_ms": 4.906}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2155}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4069.047, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562adca98a40] error while decoding MB 108 36, bytestream -33 [in#0/rtsp @ 0x562adc440e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562adc440e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:24:22+09:00", "last_error_summary": "[h264 @ 0x562adca98a40] error while decoding MB 108 36, bytestream -33 [in#0/rtsp @ 0x562adc440e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562adc440e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:19:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 319.703, "last_started_at": "2026-07-01T22:19:03+09:00", "pid": 26358, "process": {"exists": false, "pid": 26358}, "restart_count": 24, "restart_count_per_hour": 9, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.708, "sample_count": 2573, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 258648, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:24:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T22:24:22+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-01T22:24:22+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T22:24:22.790656+09:00", "mtime_epoch": 1782912262.790656, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101918}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:24:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 987.199, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 35, "pushed_frames": 35, "read_frames": 36, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.062, "pushed": 2.062, "read": 2.121}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.495, "count": 35, "max_ms": 5.756, "p95_ms": 4.204}, "gstreamer_push": {"avg_ms": 1.353, "count": 35, "max_ms": 1.698, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.26, "count": 35, "max_ms": 76.058, "p95_ms": 74.923}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.108, "count": 35, "max_ms": 76.652, "p95_ms": 76.62}, "yolo11_640_fire_11class": {"avg_ms": 65.23, "count": 35, "max_ms": 76.07, "p95_ms": 75.66}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.772, "count": 35, "max_ms": 87.445, "p95_ms": 85.186}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.633, "count": 35, "max_ms": 99.682, "p95_ms": 97.859}, "read_interval": {"avg_ms": 943.052, "count": 36, "max_ms": 17042.119, "p95_ms": 13556.028}, "triton_wait": {"avg_ms": 70.989, "count": 35, "max_ms": 87.457, "p95_ms": 85.198}, "visualize": {"avg_ms": 3.854, "count": 35, "max_ms": 5.869, "p95_ms": 5.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.971, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2155}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4068.57, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:24:22+09:00", "last_error_summary": "[h264 @ 0x562adca98a40] error while decoding MB 108 36, bytestream -33 [in#0/rtsp @ 0x562adc440e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562adc440e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:24:22+09:00", "last_error_summary": "[h264 @ 0x562adca98a40] error while decoding MB 108 36, bytestream -33 [in#0/rtsp @ 0x562adc440e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562adc440e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:24:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 319.748, "last_started_at": "2026-07-01T22:19:03+09:00", "pid": 26358, "process": {"exists": false, "pid": 26358}, "restart_count": 24, "restart_count_per_hour": 9, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.708, "sample_count": 2573, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 258649, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:24:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T22:24:22+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-01T22:24:22+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-01T22:24:22.790656+09:00", "mtime_epoch": 1782912262.790656, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101918}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:24:22+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 987.203, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 36, "pushed_frames": 36, "read_frames": 36, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.116, "pushed": 2.116, "read": 2.116}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.491, "count": 36, "max_ms": 5.756, "p95_ms": 4.204}, "gstreamer_push": {"avg_ms": 1.351, "count": 36, "max_ms": 1.698, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.47, "count": 36, "max_ms": 76.058, "p95_ms": 74.923}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.385, "count": 36, "max_ms": 76.652, "p95_ms": 76.62}, "yolo11_640_fire_11class": {"avg_ms": 65.473, "count": 36, "max_ms": 76.07, "p95_ms": 75.66}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.136, "count": 36, "max_ms": 87.445, "p95_ms": 85.186}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.982, "count": 36, "max_ms": 99.682, "p95_ms": 97.859}, "read_interval": {"avg_ms": 943.052, "count": 36, "max_ms": 17042.119, "p95_ms": 13556.028}, "triton_wait": {"avg_ms": 71.347, "count": 36, "max_ms": 87.457, "p95_ms": 85.198}, "visualize": {"avg_ms": 3.849, "count": 36, "max_ms": 5.869, "p95_ms": 5.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.015, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2155}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4068.652, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:24:22+09:00", "last_error_summary": "[h264 @ 0x562adca98a40] error while decoding MB 108 36, bytestream -33 [in#0/rtsp @ 0x562adc440e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562adc440e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:24:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 319.755, "last_started_at": "2026-07-01T22:19:03+09:00", "pid": 26358, "process": {"exists": false, "pid": 26358}, "restart_count": 24, "restart_count_per_hour": 9, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.708, "sample_count": 2573, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 258649, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:24:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-01T22:24:22+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-01T22:24:22+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-01T22:24:22.790656+09:00", "mtime_epoch": 1782912262.790656, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101918}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:24:22+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 987.203, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 36, "pushed_frames": 36, "read_frames": 36, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.115, "pushed": 2.115, "read": 2.115}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.491, "count": 36, "max_ms": 5.756, "p95_ms": 4.204}, "gstreamer_push": {"avg_ms": 1.351, "count": 36, "max_ms": 1.698, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.47, "count": 36, "max_ms": 76.058, "p95_ms": 74.923}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.385, "count": 36, "max_ms": 76.652, "p95_ms": 76.62}, "yolo11_640_fire_11class": {"avg_ms": 65.473, "count": 36, "max_ms": 76.07, "p95_ms": 75.66}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.136, "count": 36, "max_ms": 87.445, "p95_ms": 85.186}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.982, "count": 36, "max_ms": 99.682, "p95_ms": 97.859}, "read_interval": {"avg_ms": 943.052, "count": 36, "max_ms": 17042.119, "p95_ms": 13556.028}, "triton_wait": {"avg_ms": 71.347, "count": 36, "max_ms": 87.457, "p95_ms": 85.198}, "visualize": {"avg_ms": 3.849, "count": 36, "max_ms": 5.869, "p95_ms": 5.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.022, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2155}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4132.293, "limit_mb": 8796093022207.996}, "detail": {"pid": 26724, "restart_count": 25, "start_count": 26}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:24:22+09:00", "last_error_summary": "[h264 @ 0x562adca98a40] error while decoding MB 108 36, bytestream -33 [in#0/rtsp @ 0x562adc440e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562adc440e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:24:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:24:25+09:00", "pid": 26724, "process": {"exists": true, "fd_count": 5, "hwm_mb": 96.367, "name": "ffmpeg", "pid": 26724, "rss_mb": 96.367, "state": "S (sleeping)", "threads": 19}, "restart_count": 25, "restart_count_per_hour": 10, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.761, "sample_count": 2525, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 258649, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:24:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.762, "last_frame_pushed_at": "2026-07-01T22:24:22+09:00", "last_frame_read_age_sec": 4.86, "last_frame_read_at": "2026-07-01T22:24:22+09:00", "mjpeg": {"age_sec": 4.745, "exists": true, "mtime": "2026-07-01T22:24:22.902658+09:00", "mtime_epoch": 1782912262.9026585, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95920}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:24:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 987.203, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 36, "pushed_frames": 36, "read_frames": 36, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.654, "pushed": 1.654, "read": 1.654}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.491, "count": 36, "max_ms": 5.756, "p95_ms": 4.204}, "gstreamer_push": {"avg_ms": 1.351, "count": 36, "max_ms": 1.698, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.47, "count": 36, "max_ms": 76.058, "p95_ms": 74.923}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.385, "count": 36, "max_ms": 76.652, "p95_ms": 76.62}, "yolo11_640_fire_11class": {"avg_ms": 65.473, "count": 36, "max_ms": 76.07, "p95_ms": 75.66}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.136, "count": 36, "max_ms": 87.445, "p95_ms": 85.186}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.982, "count": 36, "max_ms": 99.682, "p95_ms": 97.859}, "read_interval": {"avg_ms": 943.052, "count": 36, "max_ms": 17042.119, "p95_ms": 13556.028}, "triton_wait": {"avg_ms": 71.347, "count": 36, "max_ms": 87.457, "p95_ms": 85.198}, "visualize": {"avg_ms": 3.849, "count": 36, "max_ms": 5.869, "p95_ms": 5.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.768, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2170}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4085.098, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x561819d49200] missing picture in access unit with size 18 [h264 @ 0x561819ebad80] no frame! [in#0/rtsp @ 0x561819d44e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561819d44e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561819d49840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:28:09+09:00", "last_error_summary": "[NULL @ 0x561819d49200] missing picture in access unit with size 18 [h264 @ 0x561819ebad80] no frame! [in#0/rtsp @ 0x561819d44e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561819d44e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561819d49840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:24:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 224.343, "last_started_at": "2026-07-01T22:24:25+09:00", "pid": 26724, "process": {"exists": false, "pid": 26724}, "restart_count": 25, "restart_count_per_hour": 8, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.947, "sample_count": 2283, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 260439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:28:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-01T22:28:09+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-01T22:28:09+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T22:28:09.927413+09:00", "mtime_epoch": 1782912489.9274135, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93062}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:28:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.07, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 40, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.334, "pushed": 1.334, "read": 1.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.406, "count": 40, "max_ms": 4.075, "p95_ms": 3.724}, "gstreamer_push": {"avg_ms": 1.66, "count": 40, "max_ms": 4.317, "p95_ms": 3.79}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.941, "count": 40, "max_ms": 73.88, "p95_ms": 70.216}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.629, "count": 40, "max_ms": 78.288, "p95_ms": 76.017}, "yolo11_640_fire_11class": {"avg_ms": 65.329, "count": 40, "max_ms": 76.745, "p95_ms": 74.623}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.175, "count": 40, "max_ms": 86.797, "p95_ms": 83.582}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.561, "count": 40, "max_ms": 98.734, "p95_ms": 95.766}, "read_interval": {"avg_ms": 750.138, "count": 40, "max_ms": 16315.487, "p95_ms": 115.506}, "triton_wait": {"avg_ms": 72.228, "count": 40, "max_ms": 86.809, "p95_ms": 83.593}, "visualize": {"avg_ms": 3.685, "count": 40, "max_ms": 4.588, "p95_ms": 3.934}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2170}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4084.77, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:28:09+09:00", "last_error_summary": "[NULL @ 0x561819d49200] missing picture in access unit with size 18 [h264 @ 0x561819ebad80] no frame! [in#0/rtsp @ 0x561819d44e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561819d44e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561819d49840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:28:09+09:00", "last_error_summary": "[NULL @ 0x561819d49200] missing picture in access unit with size 18 [h264 @ 0x561819ebad80] no frame! [in#0/rtsp @ 0x561819d44e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561819d44e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561819d49840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:28:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 224.363, "last_started_at": "2026-07-01T22:24:25+09:00", "pid": 26724, "process": {"exists": false, "pid": 26724}, "restart_count": 25, "restart_count_per_hour": 8, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.947, "sample_count": 2284, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 260440, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:28:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T22:28:10+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-01T22:28:09+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-01T22:28:09.927413+09:00", "mtime_epoch": 1782912489.9274135, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93062}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:28:10+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 41, "pushed_frames": 41, "read_frames": 40, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.367, "pushed": 1.367, "read": 1.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.406, "count": 41, "max_ms": 4.075, "p95_ms": 3.724}, "gstreamer_push": {"avg_ms": 1.654, "count": 41, "max_ms": 4.317, "p95_ms": 3.79}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.65, "count": 41, "max_ms": 73.88, "p95_ms": 70.216}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.451, "count": 41, "max_ms": 78.288, "p95_ms": 76.017}, "yolo11_640_fire_11class": {"avg_ms": 65.145, "count": 41, "max_ms": 76.745, "p95_ms": 74.623}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.839, "count": 41, "max_ms": 86.797, "p95_ms": 83.582}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.389, "count": 41, "max_ms": 98.734, "p95_ms": 95.766}, "read_interval": {"avg_ms": 750.138, "count": 40, "max_ms": 16315.487, "p95_ms": 115.506}, "triton_wait": {"avg_ms": 71.914, "count": 41, "max_ms": 86.809, "p95_ms": 83.593}, "visualize": {"avg_ms": 3.752, "count": 41, "max_ms": 6.443, "p95_ms": 4.15}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2170}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4084.816, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:28:09+09:00", "last_error_summary": "[NULL @ 0x561819d49200] missing picture in access unit with size 18 [h264 @ 0x561819ebad80] no frame! [in#0/rtsp @ 0x561819d44e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561819d44e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561819d49840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:28:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 224.371, "last_started_at": "2026-07-01T22:24:25+09:00", "pid": 26724, "process": {"exists": false, "pid": 26724}, "restart_count": 25, "restart_count_per_hour": 8, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.947, "sample_count": 2284, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 260440, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:28:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T22:28:10+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T22:28:09+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-01T22:28:09.927413+09:00", "mtime_epoch": 1782912489.9274135, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93062}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:28:10+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 41, "read_frames": 40, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.333, "pushed": 1.367, "read": 1.333}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.4, "count": 40, "max_ms": 4.075, "p95_ms": 3.724}, "gstreamer_push": {"avg_ms": 1.654, "count": 41, "max_ms": 4.317, "p95_ms": 3.79}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.511, "count": 40, "max_ms": 73.88, "p95_ms": 70.216}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.264, "count": 40, "max_ms": 78.288, "p95_ms": 76.017}, "yolo11_640_fire_11class": {"avg_ms": 64.906, "count": 40, "max_ms": 76.745, "p95_ms": 74.13}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.574, "count": 40, "max_ms": 86.797, "p95_ms": 83.582}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.116, "count": 40, "max_ms": 98.734, "p95_ms": 95.766}, "read_interval": {"avg_ms": 750.138, "count": 40, "max_ms": 16315.487, "p95_ms": 115.506}, "triton_wait": {"avg_ms": 71.651, "count": 40, "max_ms": 86.809, "p95_ms": 83.593}, "visualize": {"avg_ms": 3.753, "count": 40, "max_ms": 6.443, "p95_ms": 4.15}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2170}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4210.445, "limit_mb": 8796093022207.996}, "detail": {"pid": 26876, "restart_count": 26, "start_count": 27}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:28:09+09:00", "last_error_summary": "[NULL @ 0x561819d49200] missing picture in access unit with size 18 [h264 @ 0x561819ebad80] no frame! [in#0/rtsp @ 0x561819d44e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561819d44e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x561819d49840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:28:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:28:12+09:00", "pid": 26876, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.77, "name": "ffmpeg", "pid": 26876, "rss_mb": 156.605, "state": "R (running)", "threads": 49}, "restart_count": 26, "restart_count_per_hour": 9, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.987, "sample_count": 2223, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 260440, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:28:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.857, "last_frame_pushed_at": "2026-07-01T22:28:10+09:00", "last_frame_read_age_sec": 4.936, "last_frame_read_at": "2026-07-01T22:28:09+09:00", "mjpeg": {"age_sec": 4.84, "exists": true, "mtime": "2026-07-01T22:28:10.019415+09:00", "mtime_epoch": 1782912490.0194154, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94477}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:28:14+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.07, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.587, "pushed": 0.587, "read": 0.587}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.386, "count": 13, "max_ms": 3.724, "p95_ms": 3.724}, "gstreamer_push": {"avg_ms": 1.595, "count": 13, "max_ms": 4.317, "p95_ms": 4.317}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.555, "count": 13, "max_ms": 55.354, "p95_ms": 55.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.918, "count": 13, "max_ms": 59.34, "p95_ms": 59.34}, "yolo11_640_fire_11class": {"avg_ms": 51.206, "count": 13, "max_ms": 57.797, "p95_ms": 57.797}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.896, "count": 13, "max_ms": 58.838, "p95_ms": 58.838}}, "truncated_count": 0}, "processing_total": {"avg_ms": 66.063, "count": 13, "max_ms": 77.528, "p95_ms": 77.528}, "read_interval": {"avg_ms": 2080.071, "count": 13, "max_ms": 16315.487, "p95_ms": 16315.487}, "triton_wait": {"avg_ms": 53.106, "count": 13, "max_ms": 59.35, "p95_ms": 59.35}, "visualize": {"avg_ms": 3.987, "count": 13, "max_ms": 6.443, "p95_ms": 6.443}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.134, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4104.348, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ec0ef3c3c0] error while decoding MB 107 18, bytestream -5 [in#0/rtsp @ 0x55ec0edbae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ec0edbae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:30:46+09:00", "last_error_summary": "[h264 @ 0x55ec0ef3c3c0] error while decoding MB 107 18, bytestream -5 [in#0/rtsp @ 0x55ec0edbae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ec0edbae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:28:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.424, "last_started_at": "2026-07-01T22:28:12+09:00", "pid": 26876, "process": {"exists": false, "pid": 26876}, "restart_count": 26, "restart_count_per_hour": 8, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.282, "sample_count": 2151, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261519, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:30:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T22:30:46+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-01T22:30:46+09:00", "mjpeg": {"age_sec": 0.065, "exists": true, "mtime": "2026-07-01T22:30:46.218517+09:00", "mtime_epoch": 1782912646.2185166, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95459}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:30:46+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.387, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.751, "pushed": 0.751, "read": 0.751}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.303, "count": 13, "max_ms": 3.443, "p95_ms": 3.443}, "gstreamer_push": {"avg_ms": 1.361, "count": 13, "max_ms": 1.428, "p95_ms": 1.428}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.528, "count": 13, "max_ms": 46.844, "p95_ms": 46.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.454, "count": 13, "max_ms": 77.903, "p95_ms": 77.903}, "yolo11_640_fire_11class": {"avg_ms": 48.634, "count": 13, "max_ms": 73.003, "p95_ms": 73.003}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.683, "count": 13, "max_ms": 76.135, "p95_ms": 76.135}}, "truncated_count": 0}, "processing_total": {"avg_ms": 62.073, "count": 13, "max_ms": 88.123, "p95_ms": 88.123}, "read_interval": {"avg_ms": 2589.684, "count": 13, "max_ms": 16431.775, "p95_ms": 16431.775}, "triton_wait": {"avg_ms": 50.125, "count": 13, "max_ms": 77.912, "p95_ms": 77.912}, "visualize": {"avg_ms": 3.607, "count": 13, "max_ms": 3.757, "p95_ms": 3.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.314, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4104.348, "limit_mb": 8796093022207.996}, "detail": {"actual_bytes": 0, "expected_bytes": 2764800}, "event_type": "invalid_frame_read_size", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:30:46+09:00", "last_error_summary": "[h264 @ 0x55ec0ef3c3c0] error while decoding MB 107 18, bytestream -5 [in#0/rtsp @ 0x55ec0edbae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ec0edbae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:28:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.427, "last_started_at": "2026-07-01T22:28:12+09:00", "pid": 26876, "process": {"exists": false, "pid": 26876}, "restart_count": 26, "restart_count_per_hour": 8, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.282, "sample_count": 2151, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261519, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:30:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-01T22:30:46+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-01T22:30:46+09:00", "mjpeg": {"age_sec": 0.068, "exists": true, "mtime": "2026-07-01T22:30:46.218517+09:00", "mtime_epoch": 1782912646.2185166, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95459}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:30:46+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.387, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.751, "pushed": 0.751, "read": 0.751}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.303, "count": 13, "max_ms": 3.443, "p95_ms": 3.443}, "gstreamer_push": {"avg_ms": 1.361, "count": 13, "max_ms": 1.428, "p95_ms": 1.428}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.528, "count": 13, "max_ms": 46.844, "p95_ms": 46.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.454, "count": 13, "max_ms": 77.903, "p95_ms": 77.903}, "yolo11_640_fire_11class": {"avg_ms": 48.634, "count": 13, "max_ms": 73.003, "p95_ms": 73.003}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.683, "count": 13, "max_ms": 76.135, "p95_ms": 76.135}}, "truncated_count": 0}, "processing_total": {"avg_ms": 62.073, "count": 13, "max_ms": 88.123, "p95_ms": 88.123}, "read_interval": {"avg_ms": 2589.684, "count": 13, "max_ms": 16431.775, "p95_ms": 16431.775}, "triton_wait": {"avg_ms": 50.125, "count": 13, "max_ms": 77.912, "p95_ms": 77.912}, "visualize": {"avg_ms": 3.607, "count": 13, "max_ms": 3.757, "p95_ms": 3.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.317, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4104.359, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:30:46+09:00", "last_error_summary": "[h264 @ 0x55ec0ef3c3c0] error while decoding MB 107 18, bytestream -5 [in#0/rtsp @ 0x55ec0edbae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ec0edbae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:28:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 153.441, "last_started_at": "2026-07-01T22:28:12+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 26, "restart_count_per_hour": 8, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.282, "sample_count": 2151, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261519, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:30:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.023, "last_frame_pushed_at": "2026-07-01T22:30:46+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-01T22:30:46+09:00", "mjpeg": {"age_sec": 0.009, "exists": true, "mtime": "2026-07-01T22:30:46.290518+09:00", "mtime_epoch": 1782912646.290518, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96232}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:30:46+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.387, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.75, "pushed": 0.75, "read": 0.75}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.303, "count": 13, "max_ms": 3.443, "p95_ms": 3.443}, "gstreamer_push": {"avg_ms": 1.361, "count": 13, "max_ms": 1.428, "p95_ms": 1.428}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.528, "count": 13, "max_ms": 46.844, "p95_ms": 46.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.454, "count": 13, "max_ms": 77.903, "p95_ms": 77.903}, "yolo11_640_fire_11class": {"avg_ms": 48.634, "count": 13, "max_ms": 73.003, "p95_ms": 73.003}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.683, "count": 13, "max_ms": 76.135, "p95_ms": 76.135}}, "truncated_count": 0}, "processing_total": {"avg_ms": 62.073, "count": 13, "max_ms": 88.123, "p95_ms": 88.123}, "read_interval": {"avg_ms": 2589.684, "count": 13, "max_ms": 16431.775, "p95_ms": 16431.775}, "triton_wait": {"avg_ms": 50.125, "count": 13, "max_ms": 77.912, "p95_ms": 77.912}, "visualize": {"avg_ms": 3.607, "count": 13, "max_ms": 3.757, "p95_ms": 3.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.33, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2179}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4280.355, "limit_mb": 8796093022207.996}, "detail": {"pid": 27028, "restart_count": 27, "start_count": 28}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:30:46+09:00", "last_error_summary": "[h264 @ 0x55ec0ef3c3c0] error while decoding MB 107 18, bytestream -5 [in#0/rtsp @ 0x55ec0edbae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ec0edbae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:28:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:30:49+09:00", "pid": 27028, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.301, "name": "ffmpeg", "pid": 27028, "rss_mb": 206.301, "state": "S (sleeping)", "threads": 106}, "restart_count": 27, "restart_count_per_hour": 9, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.318, "sample_count": 2108, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261519, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:30:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.746, "last_frame_pushed_at": "2026-07-01T22:30:46+09:00", "last_frame_read_age_sec": 4.809, "last_frame_read_at": "2026-07-01T22:30:46+09:00", "mjpeg": {"age_sec": 4.733, "exists": true, "mtime": "2026-07-01T22:30:46.290518+09:00", "mtime_epoch": 1782912646.290518, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96232}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:30:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.387, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.589, "pushed": 0.589, "read": 0.589}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.303, "count": 13, "max_ms": 3.443, "p95_ms": 3.443}, "gstreamer_push": {"avg_ms": 1.361, "count": 13, "max_ms": 1.428, "p95_ms": 1.428}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.528, "count": 13, "max_ms": 46.844, "p95_ms": 46.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.454, "count": 13, "max_ms": 77.903, "p95_ms": 77.903}, "yolo11_640_fire_11class": {"avg_ms": 48.634, "count": 13, "max_ms": 73.003, "p95_ms": 73.003}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.683, "count": 13, "max_ms": 76.135, "p95_ms": 76.135}}, "truncated_count": 0}, "processing_total": {"avg_ms": 62.073, "count": 13, "max_ms": 88.123, "p95_ms": 88.123}, "read_interval": {"avg_ms": 2589.684, "count": 13, "max_ms": 16431.775, "p95_ms": 16431.775}, "triton_wait": {"avg_ms": 50.125, "count": 13, "max_ms": 77.912, "p95_ms": 77.912}, "visualize": {"avg_ms": 3.607, "count": 13, "max_ms": 3.757, "p95_ms": 3.757}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.054, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1229}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4133.348, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5573014ae140] cabac decode of qscale diff failed at 57 32 [h264 @ 0x5573014ae140] error while decoding MB 57 32, bytestream 5375 [h264 @ 0x5573015b4d40] cabac decode of qscale diff failed at 65 27 [h264 @ 0x5573015b4d40] error while decoding MB 65 27, bytestream 5777 [h264 @ 0x557302243980] error while decoding MB 44 26, bytestream -5 [h264 @ 0x557301fd6e00] top block unavailable for requested intra mode [h264 @ 0x557301fd6e00] error while decoding MB 32 0, bytestream 1349 [h264 @ 0x55", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 166, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:30:57+09:00", "last_error_summary": "[h264 @ 0x5573014ae140] cabac decode of qscale diff failed at 57 32 [h264 @ 0x5573014ae140] error while decoding MB 57 32, bytestream 5375 [h264 @ 0x5573015b4d40] cabac decode of qscale diff failed at 65 27 [h264 @ 0x5573015b4d40] error while decoding MB 65 27, bytestream 5777 [h264 @ 0x557302243980] error while decoding MB 44 26, bytestream -5 [h264 @ 0x557301fd6e00] top block unavailable for requested intra mode [h264 @ 0x557301fd6e00] error while decoding MB 32 0, bytestream 1349 [h264 @ 0x55", "last_exit_at": "2026-07-01T22:23:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 444.594, "last_started_at": "2026-07-01T22:23:32+09:00", "pid": 26617, "process": {"exists": false, "pid": 26617}, "restart_count": 190, "restart_count_per_hour": 7, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 191}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.142, "sample_count": 2762, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 147514, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:30:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-01T22:30:57+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-01T22:30:57+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-01T22:30:57.274732+09:00", "mtime_epoch": 1782912657.274732, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 102532}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:30:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 932.594, "name": "celery", "pid": 237, "rss_mb": 929.508, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 138, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.6, "pushed": 4.6, "read": 4.6}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.528, "count": 137, "max_ms": 2.563, "p95_ms": 1.564}, "gstreamer_push": {"avg_ms": 1.663, "count": 138, "max_ms": 6.608, "p95_ms": 4.053}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.426, "count": 137, "max_ms": 77.881, "p95_ms": 53.915}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.77, "count": 137, "max_ms": 79.336, "p95_ms": 59.448}, "yolo11_640_fire_11class": {"avg_ms": 48.193, "count": 137, "max_ms": 77.115, "p95_ms": 58.064}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.156, "count": 137, "max_ms": 85.774, "p95_ms": 58.81}}, "truncated_count": 0}, "processing_total": {"avg_ms": 59.717, "count": 138, "max_ms": 96.031, "p95_ms": 69.666}, "read_interval": {"avg_ms": 217.41, "count": 138, "max_ms": 19516.834, "p95_ms": 95.902}, "triton_wait": {"avg_ms": 49.551, "count": 137, "max_ms": 85.785, "p95_ms": 59.46}, "visualize": {"avg_ms": 3.689, "count": 137, "max_ms": 4.462, "p95_ms": 3.897}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1229}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4132.645, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:30:57+09:00", "last_error_summary": "[h264 @ 0x5573014ae140] cabac decode of qscale diff failed at 57 32 [h264 @ 0x5573014ae140] error while decoding MB 57 32, bytestream 5375 [h264 @ 0x5573015b4d40] cabac decode of qscale diff failed at 65 27 [h264 @ 0x5573015b4d40] error while decoding MB 65 27, bytestream 5777 [h264 @ 0x557302243980] error while decoding MB 44 26, bytestream -5 [h264 @ 0x557301fd6e00] top block unavailable for requested intra mode [h264 @ 0x557301fd6e00] error while decoding MB 32 0, bytestream 1349 [h264 @ 0x55", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 166, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:30:57+09:00", "last_error_summary": "[h264 @ 0x5573014ae140] cabac decode of qscale diff failed at 57 32 [h264 @ 0x5573014ae140] error while decoding MB 57 32, bytestream 5375 [h264 @ 0x5573015b4d40] cabac decode of qscale diff failed at 65 27 [h264 @ 0x5573015b4d40] error while decoding MB 65 27, bytestream 5777 [h264 @ 0x557302243980] error while decoding MB 44 26, bytestream -5 [h264 @ 0x557301fd6e00] top block unavailable for requested intra mode [h264 @ 0x557301fd6e00] error while decoding MB 32 0, bytestream 1349 [h264 @ 0x55", "last_exit_at": "2026-07-01T22:30:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 444.63, "last_started_at": "2026-07-01T22:23:32+09:00", "pid": 26617, "process": {"exists": false, "pid": 26617}, "restart_count": 190, "restart_count_per_hour": 7, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 191}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.142, "sample_count": 2763, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 147515, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:30:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T22:30:57+09:00", "last_frame_read_age_sec": 0.087, "last_frame_read_at": "2026-07-01T22:30:57+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-01T22:30:57.274732+09:00", "mtime_epoch": 1782912657.274732, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 102532}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:30:57+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 932.594, "name": "celery", "pid": 237, "rss_mb": 929.508, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.607, "pushed": 4.607, "read": 4.573}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.528, "count": 138, "max_ms": 2.563, "p95_ms": 1.564}, "gstreamer_push": {"avg_ms": 1.645, "count": 138, "max_ms": 6.608, "p95_ms": 4.053}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.469, "count": 138, "max_ms": 77.881, "p95_ms": 53.915}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.883, "count": 138, "max_ms": 79.336, "p95_ms": 64.304}, "yolo11_640_fire_11class": {"avg_ms": 48.301, "count": 138, "max_ms": 77.115, "p95_ms": 63.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.262, "count": 138, "max_ms": 85.774, "p95_ms": 63.795}}, "truncated_count": 0}, "processing_total": {"avg_ms": 59.844, "count": 138, "max_ms": 96.031, "p95_ms": 74.426}, "read_interval": {"avg_ms": 218.448, "count": 137, "max_ms": 19516.834, "p95_ms": 95.902}, "triton_wait": {"avg_ms": 49.658, "count": 138, "max_ms": 85.785, "p95_ms": 64.313}, "visualize": {"avg_ms": 3.69, "count": 138, "max_ms": 4.462, "p95_ms": 3.897}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1229}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4132.535, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 166, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:30:57+09:00", "last_error_summary": "[h264 @ 0x5573014ae140] cabac decode of qscale diff failed at 57 32 [h264 @ 0x5573014ae140] error while decoding MB 57 32, bytestream 5375 [h264 @ 0x5573015b4d40] cabac decode of qscale diff failed at 65 27 [h264 @ 0x5573015b4d40] error while decoding MB 65 27, bytestream 5777 [h264 @ 0x557302243980] error while decoding MB 44 26, bytestream -5 [h264 @ 0x557301fd6e00] top block unavailable for requested intra mode [h264 @ 0x557301fd6e00] error while decoding MB 32 0, bytestream 1349 [h264 @ 0x55", "last_exit_at": "2026-07-01T22:30:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 444.638, "last_started_at": "2026-07-01T22:23:32+09:00", "pid": 26617, "process": {"exists": false, "pid": 26617}, "restart_count": 190, "restart_count_per_hour": 7, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 191}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.142, "sample_count": 2763, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 147515, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:30:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-01T22:30:57+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-01T22:30:57+09:00", "mjpeg": {"age_sec": 0.004, "exists": true, "mtime": "2026-07-01T22:30:57.362734+09:00", "mtime_epoch": 1782912657.3627338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 101721}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:30:57+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 932.594, "name": "celery", "pid": 237, "rss_mb": 929.508, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.605, "pushed": 4.605, "read": 4.572}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.528, "count": 138, "max_ms": 2.563, "p95_ms": 1.564}, "gstreamer_push": {"avg_ms": 1.645, "count": 138, "max_ms": 6.608, "p95_ms": 4.053}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.469, "count": 138, "max_ms": 77.881, "p95_ms": 53.915}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.883, "count": 138, "max_ms": 79.336, "p95_ms": 64.304}, "yolo11_640_fire_11class": {"avg_ms": 48.301, "count": 138, "max_ms": 77.115, "p95_ms": 63.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.262, "count": 138, "max_ms": 85.774, "p95_ms": 63.795}}, "truncated_count": 0}, "processing_total": {"avg_ms": 59.844, "count": 138, "max_ms": 96.031, "p95_ms": 74.426}, "read_interval": {"avg_ms": 218.448, "count": 137, "max_ms": 19516.834, "p95_ms": 95.902}, "triton_wait": {"avg_ms": 49.658, "count": 138, "max_ms": 85.785, "p95_ms": 64.313}, "visualize": {"avg_ms": 3.69, "count": 138, "max_ms": 4.462, "p95_ms": 3.897}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1229}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4241.863, "limit_mb": 8796093022207.996}, "detail": {"pid": 27180, "restart_count": 191, "start_count": 192}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 166, "h264_decode_error_count": 214, "h264_decode_error_count_per_hour": 39, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:30:57+09:00", "last_error_summary": "[h264 @ 0x5573014ae140] cabac decode of qscale diff failed at 57 32 [h264 @ 0x5573014ae140] error while decoding MB 57 32, bytestream 5375 [h264 @ 0x5573015b4d40] cabac decode of qscale diff failed at 65 27 [h264 @ 0x5573015b4d40] error while decoding MB 65 27, bytestream 5777 [h264 @ 0x557302243980] error while decoding MB 44 26, bytestream -5 [h264 @ 0x557301fd6e00] top block unavailable for requested intra mode [h264 @ 0x557301fd6e00] error while decoding MB 32 0, bytestream 1349 [h264 @ 0x55", "last_exit_at": "2026-07-01T22:31:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:31:00+09:00", "pid": 27180, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.363, "name": "ffmpeg", "pid": 27180, "rss_mb": 139.363, "state": "S (sleeping)", "threads": 106}, "restart_count": 191, "restart_count_per_hour": 8, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 192}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.153, "sample_count": 2720, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 147515, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:30:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.714, "last_frame_pushed_at": "2026-07-01T22:30:57+09:00", "last_frame_read_age_sec": 4.791, "last_frame_read_at": "2026-07-01T22:30:57+09:00", "mjpeg": {"age_sec": 4.7, "exists": true, "mtime": "2026-07-01T22:30:57.362734+09:00", "mtime_epoch": 1782912657.3627338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 101721}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:31:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 932.594, "name": "celery", "pid": 237, "rss_mb": 929.508, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 75, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.533, "pushed": 2.533, "read": 2.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.525, "count": 75, "max_ms": 2.563, "p95_ms": 1.56}, "gstreamer_push": {"avg_ms": 1.555, "count": 76, "max_ms": 6.296, "p95_ms": 3.675}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.841, "count": 75, "max_ms": 77.881, "p95_ms": 73.803}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.169, "count": 75, "max_ms": 79.336, "p95_ms": 74.932}, "yolo11_640_fire_11class": {"avg_ms": 49.565, "count": 75, "max_ms": 77.115, "p95_ms": 74.447}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.672, "count": 75, "max_ms": 85.774, "p95_ms": 82.699}}, "truncated_count": 0}, "processing_total": {"avg_ms": 61.165, "count": 76, "max_ms": 96.031, "p95_ms": 92.79}, "read_interval": {"avg_ms": 336.877, "count": 75, "max_ms": 19516.834, "p95_ms": 108.924}, "triton_wait": {"avg_ms": 51.064, "count": 75, "max_ms": 85.785, "p95_ms": 82.711}, "visualize": {"avg_ms": 3.704, "count": 75, "max_ms": 4.462, "p95_ms": 3.921}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4086.41, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a73377b2c0] error while decoding MB 118 33, bytestream -19 [in#0/rtsp @ 0x55a7331efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a7331efe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:33:54+09:00", "last_error_summary": "[h264 @ 0x55a73377b2c0] error while decoding MB 118 33, bytestream -19 [in#0/rtsp @ 0x55a7331efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a7331efe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:28:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.543, "last_started_at": "2026-07-01T22:30:49+09:00", "pid": 27028, "process": {"exists": false, "pid": 27028}, "restart_count": 27, "restart_count_per_hour": 8, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.673, "sample_count": 2129, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 262927, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:33:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T22:33:54+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T22:33:54+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-01T22:33:54.506128+09:00", "mtime_epoch": 1782912834.5061283, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95248}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:33:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 992.102, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.694, "pushed": 0.694, "read": 0.752}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.435, "count": 12, "max_ms": 3.78, "p95_ms": 3.78}, "gstreamer_push": {"avg_ms": 1.549, "count": 12, "max_ms": 3.963, "p95_ms": 3.963}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.903, "count": 12, "max_ms": 50.139, "p95_ms": 50.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.719, "count": 12, "max_ms": 56.449, "p95_ms": 56.449}, "yolo11_640_fire_11class": {"avg_ms": 52.371, "count": 12, "max_ms": 58.529, "p95_ms": 58.529}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.166, "count": 12, "max_ms": 56.854, "p95_ms": 56.854}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.989, "count": 12, "max_ms": 70.824, "p95_ms": 70.824}, "read_interval": {"avg_ms": 2369.878, "count": 13, "max_ms": 16341.287, "p95_ms": 16341.287}, "triton_wait": {"avg_ms": 53.691, "count": 12, "max_ms": 58.543, "p95_ms": 58.543}, "visualize": {"avg_ms": 3.684, "count": 12, "max_ms": 3.766, "p95_ms": 3.766}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.288, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4086.238, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:33:54+09:00", "last_error_summary": "[h264 @ 0x55a73377b2c0] error while decoding MB 118 33, bytestream -19 [in#0/rtsp @ 0x55a7331efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a7331efe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:33:54+09:00", "last_error_summary": "[h264 @ 0x55a73377b2c0] error while decoding MB 118 33, bytestream -19 [in#0/rtsp @ 0x55a7331efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a7331efe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:33:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.561, "last_started_at": "2026-07-01T22:30:49+09:00", "pid": 27028, "process": {"exists": false, "pid": 27028}, "restart_count": 27, "restart_count_per_hour": 8, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.673, "sample_count": 2129, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 262928, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:33:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T22:33:54+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-01T22:33:54+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-01T22:33:54.506128+09:00", "mtime_epoch": 1782912834.5061283, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95248}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:33:54+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 992.102, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.751, "pushed": 0.751, "read": 0.751}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.438, "count": 13, "max_ms": 3.78, "p95_ms": 3.78}, "gstreamer_push": {"avg_ms": 1.533, "count": 13, "max_ms": 3.963, "p95_ms": 3.963}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.183, "count": 13, "max_ms": 50.139, "p95_ms": 50.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.986, "count": 13, "max_ms": 56.449, "p95_ms": 56.449}, "yolo11_640_fire_11class": {"avg_ms": 52.724, "count": 13, "max_ms": 58.529, "p95_ms": 58.529}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.4, "count": 13, "max_ms": 56.854, "p95_ms": 56.854}}, "truncated_count": 0}, "processing_total": {"avg_ms": 66.659, "count": 13, "max_ms": 74.695, "p95_ms": 74.695}, "read_interval": {"avg_ms": 2369.878, "count": 13, "max_ms": 16341.287, "p95_ms": 16341.287}, "triton_wait": {"avg_ms": 53.943, "count": 13, "max_ms": 58.543, "p95_ms": 58.543}, "visualize": {"avg_ms": 3.827, "count": 13, "max_ms": 5.543, "p95_ms": 5.543}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.305, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4086.23, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:33:54+09:00", "last_error_summary": "[h264 @ 0x55a73377b2c0] error while decoding MB 118 33, bytestream -19 [in#0/rtsp @ 0x55a7331efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a7331efe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:33:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.568, "last_started_at": "2026-07-01T22:30:49+09:00", "pid": 27028, "process": {"exists": false, "pid": 27028}, "restart_count": 27, "restart_count_per_hour": 8, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.673, "sample_count": 2129, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 262928, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:33:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T22:33:54+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-01T22:33:54+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-01T22:33:54.506128+09:00", "mtime_epoch": 1782912834.5061283, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95248}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:33:54+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 992.102, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.751, "pushed": 0.751, "read": 0.751}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.438, "count": 13, "max_ms": 3.78, "p95_ms": 3.78}, "gstreamer_push": {"avg_ms": 1.533, "count": 13, "max_ms": 3.963, "p95_ms": 3.963}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.183, "count": 13, "max_ms": 50.139, "p95_ms": 50.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.986, "count": 13, "max_ms": 56.449, "p95_ms": 56.449}, "yolo11_640_fire_11class": {"avg_ms": 52.724, "count": 13, "max_ms": 58.529, "p95_ms": 58.529}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.4, "count": 13, "max_ms": 56.854, "p95_ms": 56.854}}, "truncated_count": 0}, "processing_total": {"avg_ms": 66.659, "count": 13, "max_ms": 74.695, "p95_ms": 74.695}, "read_interval": {"avg_ms": 2369.878, "count": 13, "max_ms": 16341.287, "p95_ms": 16341.287}, "triton_wait": {"avg_ms": 53.943, "count": 13, "max_ms": 58.543, "p95_ms": 58.543}, "visualize": {"avg_ms": 3.827, "count": 13, "max_ms": 5.543, "p95_ms": 5.543}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.312, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4262.906, "limit_mb": 8796093022207.996}, "detail": {"pid": 27287, "restart_count": 28, "start_count": 29}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:33:54+09:00", "last_error_summary": "[h264 @ 0x55a73377b2c0] error while decoding MB 118 33, bytestream -19 [in#0/rtsp @ 0x55a7331efe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a7331efe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:33:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:33:57+09:00", "pid": 27287, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.672, "name": "ffmpeg", "pid": 27287, "rss_mb": 206.672, "state": "S (sleeping)", "threads": 106}, "restart_count": 28, "restart_count_per_hour": 9, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.752, "sample_count": 2088, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 262928, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:33:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.537, "last_frame_pushed_at": "2026-07-01T22:33:54+09:00", "last_frame_read_age_sec": 4.614, "last_frame_read_at": "2026-07-01T22:33:54+09:00", "mjpeg": {"age_sec": 4.521, "exists": true, "mtime": "2026-07-01T22:33:54.594130+09:00", "mtime_epoch": 1782912834.59413, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 96576}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:33:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 992.102, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.595, "pushed": 0.595, "read": 0.595}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.438, "count": 13, "max_ms": 3.78, "p95_ms": 3.78}, "gstreamer_push": {"avg_ms": 1.533, "count": 13, "max_ms": 3.963, "p95_ms": 3.963}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.183, "count": 13, "max_ms": 50.139, "p95_ms": 50.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.986, "count": 13, "max_ms": 56.449, "p95_ms": 56.449}, "yolo11_640_fire_11class": {"avg_ms": 52.724, "count": 13, "max_ms": 58.529, "p95_ms": 58.529}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.4, "count": 13, "max_ms": 56.854, "p95_ms": 56.854}}, "truncated_count": 0}, "processing_total": {"avg_ms": 66.659, "count": 13, "max_ms": 74.695, "p95_ms": 74.695}, "read_interval": {"avg_ms": 2369.878, "count": 13, "max_ms": 16341.287, "p95_ms": 16341.287}, "triton_wait": {"avg_ms": 53.943, "count": 13, "max_ms": 58.543, "p95_ms": 58.543}, "visualize": {"avg_ms": 3.827, "count": 13, "max_ms": 5.543, "p95_ms": 5.543}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.838, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2208}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4097.277, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563cbb41e580] error while decoding MB 87 2, bytestream -25 [in#0/rtsp @ 0x563cbb29ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563cbb29ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:38:07+09:00", "last_error_summary": "[h264 @ 0x563cbb41e580] error while decoding MB 87 2, bytestream -25 [in#0/rtsp @ 0x563cbb29ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563cbb29ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:33:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 250.528, "last_started_at": "2026-07-01T22:33:57+09:00", "pid": 27287, "process": {"exists": false, "pid": 27287}, "restart_count": 28, "restart_count_per_hour": 9, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.659, "sample_count": 2197, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 264966, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:38:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.084, "last_frame_pushed_at": "2026-07-01T22:38:07+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T22:38:07+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T22:38:07.578834+09:00", "mtime_epoch": 1782913087.5788343, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98121}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:38:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 993.133, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 19, "pushed_frames": 19, "read_frames": 19, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.634, "pushed": 0.634, "read": 0.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.402, "count": 19, "max_ms": 3.726, "p95_ms": 3.726}, "gstreamer_push": {"avg_ms": 1.594, "count": 19, "max_ms": 5.976, "p95_ms": 5.976}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.554, "count": 19, "max_ms": 74.444, "p95_ms": 74.444}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.054, "count": 19, "max_ms": 78.648, "p95_ms": 78.648}, "yolo11_640_fire_11class": {"avg_ms": 70.863, "count": 19, "max_ms": 76.963, "p95_ms": 76.963}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.531, "count": 19, "max_ms": 85.613, "p95_ms": 85.613}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.099, "count": 19, "max_ms": 97.808, "p95_ms": 97.808}, "read_interval": {"avg_ms": 1578.329, "count": 19, "max_ms": 16533.128, "p95_ms": 16533.128}, "triton_wait": {"avg_ms": 78.712, "count": 19, "max_ms": 85.624, "p95_ms": 85.624}, "visualize": {"avg_ms": 3.648, "count": 19, "max_ms": 3.758, "p95_ms": 3.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2208}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4096.73, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:38:07+09:00", "last_error_summary": "[h264 @ 0x563cbb41e580] error while decoding MB 87 2, bytestream -25 [in#0/rtsp @ 0x563cbb29ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563cbb29ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:38:07+09:00", "last_error_summary": "[h264 @ 0x563cbb41e580] error while decoding MB 87 2, bytestream -25 [in#0/rtsp @ 0x563cbb29ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563cbb29ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:38:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 250.561, "last_started_at": "2026-07-01T22:33:57+09:00", "pid": 27287, "process": {"exists": false, "pid": 27287}, "restart_count": 28, "restart_count_per_hour": 9, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.659, "sample_count": 2197, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 264967, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:38:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T22:38:07+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-01T22:38:07+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-01T22:38:07.578834+09:00", "mtime_epoch": 1782913087.5788343, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98121}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:38:07+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 993.133, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 20, "pushed_frames": 20, "read_frames": 19, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.667, "pushed": 0.667, "read": 0.633}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.401, "count": 19, "max_ms": 3.726, "p95_ms": 3.726}, "gstreamer_push": {"avg_ms": 1.576, "count": 20, "max_ms": 5.976, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.134, "count": 19, "max_ms": 73.713, "p95_ms": 73.713}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.92, "count": 19, "max_ms": 77.073, "p95_ms": 77.073}, "yolo11_640_fire_11class": {"avg_ms": 70.918, "count": 19, "max_ms": 76.963, "p95_ms": 76.963}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.527, "count": 19, "max_ms": 85.613, "p95_ms": 85.613}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.355, "count": 20, "max_ms": 97.808, "p95_ms": 97.797}, "read_interval": {"avg_ms": 1578.329, "count": 19, "max_ms": 16533.128, "p95_ms": 16533.128}, "triton_wait": {"avg_ms": 78.708, "count": 19, "max_ms": 85.624, "p95_ms": 85.624}, "visualize": {"avg_ms": 3.647, "count": 19, "max_ms": 3.758, "p95_ms": 3.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2208}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4096.645, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:38:07+09:00", "last_error_summary": "[h264 @ 0x563cbb41e580] error while decoding MB 87 2, bytestream -25 [in#0/rtsp @ 0x563cbb29ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563cbb29ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:38:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 250.568, "last_started_at": "2026-07-01T22:33:57+09:00", "pid": 27287, "process": {"exists": false, "pid": 27287}, "restart_count": 28, "restart_count_per_hour": 9, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.659, "sample_count": 2197, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 264967, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:38:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T22:38:07+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T22:38:07+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-01T22:38:07.578834+09:00", "mtime_epoch": 1782913087.5788343, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98121}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:38:07+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 993.133, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 19, "pushed_frames": 19, "read_frames": 19, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.634, "pushed": 0.634, "read": 0.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.401, "count": 19, "max_ms": 3.726, "p95_ms": 3.726}, "gstreamer_push": {"avg_ms": 1.587, "count": 19, "max_ms": 5.976, "p95_ms": 5.976}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.134, "count": 19, "max_ms": 73.713, "p95_ms": 73.713}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.92, "count": 19, "max_ms": 77.073, "p95_ms": 77.073}, "yolo11_640_fire_11class": {"avg_ms": 70.918, "count": 19, "max_ms": 76.963, "p95_ms": 76.963}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.527, "count": 19, "max_ms": 85.613, "p95_ms": 85.613}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.096, "count": 19, "max_ms": 97.808, "p95_ms": 97.808}, "read_interval": {"avg_ms": 1578.329, "count": 19, "max_ms": 16533.128, "p95_ms": 16533.128}, "triton_wait": {"avg_ms": 78.708, "count": 19, "max_ms": 85.624, "p95_ms": 85.624}, "visualize": {"avg_ms": 3.647, "count": 19, "max_ms": 3.758, "p95_ms": 3.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2208}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4276.117, "limit_mb": 8796093022207.996}, "detail": {"pid": 27439, "restart_count": 29, "start_count": 30}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:38:07+09:00", "last_error_summary": "[h264 @ 0x563cbb41e580] error while decoding MB 87 2, bytestream -25 [in#0/rtsp @ 0x563cbb29ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563cbb29ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:38:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:38:10+09:00", "pid": 27439, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.125, "name": "ffmpeg", "pid": 27439, "rss_mb": 209.125, "state": "S (sleeping)", "threads": 106}, "restart_count": 29, "restart_count_per_hour": 10, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.679, "sample_count": 2157, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 264967, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:38:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.464, "last_frame_pushed_at": "2026-07-01T22:38:07+09:00", "last_frame_read_age_sec": 4.562, "last_frame_read_at": "2026-07-01T22:38:07+09:00", "mjpeg": {"age_sec": 4.447, "exists": true, "mtime": "2026-07-01T22:38:07.686836+09:00", "mtime_epoch": 1782913087.6868362, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98352}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:38:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 993.133, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.583, "pushed": 0.583, "read": 0.583}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.407, "count": 13, "max_ms": 3.726, "p95_ms": 3.726}, "gstreamer_push": {"avg_ms": 1.331, "count": 13, "max_ms": 1.426, "p95_ms": 1.426}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.747, "count": 13, "max_ms": 73.713, "p95_ms": 73.713}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.855, "count": 13, "max_ms": 77.073, "p95_ms": 77.073}, "yolo11_640_fire_11class": {"avg_ms": 70.738, "count": 13, "max_ms": 76.082, "p95_ms": 76.082}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.375, "count": 13, "max_ms": 85.613, "p95_ms": 85.613}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.898, "count": 13, "max_ms": 97.808, "p95_ms": 97.808}, "read_interval": {"avg_ms": 2256.348, "count": 13, "max_ms": 16533.128, "p95_ms": 16533.128}, "triton_wait": {"avg_ms": 78.634, "count": 13, "max_ms": 85.624, "p95_ms": 85.624}, "visualize": {"avg_ms": 3.643, "count": 13, "max_ms": 3.733, "p95_ms": 3.733}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.302, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1279}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4152.59, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55bff17ab600] left block unavailable for requested intra mode [h264 @ 0x55bff17ab600] error while decoding MB 0 18, bytestream 339 [h264 @ 0x55bff237bf80] left block unavailable for requested intra mode [h264 @ 0x55bff237bf80] error while decoding MB 0 11, bytestream 2849 [h264 @ 0x55bff17ee500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bff17ee500] error while decoding MB 0 15, bytestream 3070 [h264 @ 0x55bff18c3880] top block unavailable for requested intra mod", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 167, "h264_decode_error_count": 223, "h264_decode_error_count_per_hour": 45, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:41:54+09:00", "last_error_summary": "[h264 @ 0x55bff17ab600] left block unavailable for requested intra mode [h264 @ 0x55bff17ab600] error while decoding MB 0 18, bytestream 339 [h264 @ 0x55bff237bf80] left block unavailable for requested intra mode [h264 @ 0x55bff237bf80] error while decoding MB 0 11, bytestream 2849 [h264 @ 0x55bff17ee500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bff17ee500] error while decoding MB 0 15, bytestream 3070 [h264 @ 0x55bff18c3880] top block unavailable for requested intra mod", "last_exit_at": "2026-07-01T22:31:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 654.274, "last_started_at": "2026-07-01T22:31:00+09:00", "pid": 27180, "process": {"exists": false, "pid": 27180}, "restart_count": 191, "restart_count_per_hour": 7, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 192}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.731, "sample_count": 2663, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 153557, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:41:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-01T22:41:54+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-01T22:41:54+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-01T22:41:54.282956+09:00", "mtime_epoch": 1782913314.282956, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 107590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:41:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 940.395, "name": "celery", "pid": 237, "rss_mb": 940.395, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 104, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.472, "pushed": 3.472, "read": 3.472}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.814, "count": 105, "max_ms": 2.448, "p95_ms": 2.053}, "gstreamer_push": {"avg_ms": 1.528, "count": 104, "max_ms": 4.0, "p95_ms": 3.734}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.773, "count": 105, "max_ms": 76.268, "p95_ms": 71.883}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.101, "count": 105, "max_ms": 82.118, "p95_ms": 78.178}, "yolo11_640_fire_11class": {"avg_ms": 69.401, "count": 105, "max_ms": 77.806, "p95_ms": 76.293}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.153, "count": 105, "max_ms": 86.492, "p95_ms": 84.834}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.459, "count": 104, "max_ms": 97.038, "p95_ms": 95.768}, "read_interval": {"avg_ms": 288.339, "count": 104, "max_ms": 19177.371, "p95_ms": 113.251}, "triton_wait": {"avg_ms": 77.458, "count": 105, "max_ms": 86.504, "p95_ms": 84.85}, "visualize": {"avg_ms": 3.772, "count": 105, "max_ms": 4.464, "p95_ms": 3.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1279}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4152.387, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:41:54+09:00", "last_error_summary": "[h264 @ 0x55bff17ab600] left block unavailable for requested intra mode [h264 @ 0x55bff17ab600] error while decoding MB 0 18, bytestream 339 [h264 @ 0x55bff237bf80] left block unavailable for requested intra mode [h264 @ 0x55bff237bf80] error while decoding MB 0 11, bytestream 2849 [h264 @ 0x55bff17ee500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bff17ee500] error while decoding MB 0 15, bytestream 3070 [h264 @ 0x55bff18c3880] top block unavailable for requested intra mod", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 167, "h264_decode_error_count": 223, "h264_decode_error_count_per_hour": 45, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:41:54+09:00", "last_error_summary": "[h264 @ 0x55bff17ab600] left block unavailable for requested intra mode [h264 @ 0x55bff17ab600] error while decoding MB 0 18, bytestream 339 [h264 @ 0x55bff237bf80] left block unavailable for requested intra mode [h264 @ 0x55bff237bf80] error while decoding MB 0 11, bytestream 2849 [h264 @ 0x55bff17ee500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bff17ee500] error while decoding MB 0 15, bytestream 3070 [h264 @ 0x55bff18c3880] top block unavailable for requested intra mod", "last_exit_at": "2026-07-01T22:41:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 654.292, "last_started_at": "2026-07-01T22:31:00+09:00", "pid": 27180, "process": {"exists": false, "pid": 27180}, "restart_count": 191, "restart_count_per_hour": 7, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 192}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.731, "sample_count": 2663, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 153557, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:41:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T22:41:54+09:00", "last_frame_read_age_sec": 0.074, "last_frame_read_at": "2026-07-01T22:41:54+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-01T22:41:54.282956+09:00", "mtime_epoch": 1782913314.282956, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 107590}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:41:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 940.395, "name": "celery", "pid": 237, "rss_mb": 940.395, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.504, "pushed": 3.504, "read": 3.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.814, "count": 105, "max_ms": 2.448, "p95_ms": 2.053}, "gstreamer_push": {"avg_ms": 1.527, "count": 105, "max_ms": 4.0, "p95_ms": 3.734}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.773, "count": 105, "max_ms": 76.268, "p95_ms": 71.883}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.101, "count": 105, "max_ms": 82.118, "p95_ms": 78.178}, "yolo11_640_fire_11class": {"avg_ms": 69.401, "count": 105, "max_ms": 77.806, "p95_ms": 76.293}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.153, "count": 105, "max_ms": 86.492, "p95_ms": 84.834}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.193, "count": 105, "max_ms": 97.038, "p95_ms": 95.768}, "read_interval": {"avg_ms": 288.339, "count": 104, "max_ms": 19177.371, "p95_ms": 113.251}, "triton_wait": {"avg_ms": 77.458, "count": 105, "max_ms": 86.504, "p95_ms": 84.85}, "visualize": {"avg_ms": 3.772, "count": 105, "max_ms": 4.464, "p95_ms": 3.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1279}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4152.383, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 167, "h264_decode_error_count": 223, "h264_decode_error_count_per_hour": 45, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:41:54+09:00", "last_error_summary": "[h264 @ 0x55bff17ab600] left block unavailable for requested intra mode [h264 @ 0x55bff17ab600] error while decoding MB 0 18, bytestream 339 [h264 @ 0x55bff237bf80] left block unavailable for requested intra mode [h264 @ 0x55bff237bf80] error while decoding MB 0 11, bytestream 2849 [h264 @ 0x55bff17ee500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bff17ee500] error while decoding MB 0 15, bytestream 3070 [h264 @ 0x55bff18c3880] top block unavailable for requested intra mod", "last_exit_at": "2026-07-01T22:41:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 654.3, "last_started_at": "2026-07-01T22:31:00+09:00", "pid": 27180, "process": {"exists": false, "pid": 27180}, "restart_count": 191, "restart_count_per_hour": 7, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 192}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.731, "sample_count": 2663, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 153557, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:41:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-01T22:41:54+09:00", "last_frame_read_age_sec": 0.083, "last_frame_read_at": "2026-07-01T22:41:54+09:00", "mjpeg": {"age_sec": 0.003, "exists": true, "mtime": "2026-07-01T22:41:54.358957+09:00", "mtime_epoch": 1782913314.3589573, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 105219}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:41:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 940.395, "name": "celery", "pid": 237, "rss_mb": 940.395, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.503, "pushed": 3.503, "read": 3.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.814, "count": 105, "max_ms": 2.448, "p95_ms": 2.053}, "gstreamer_push": {"avg_ms": 1.527, "count": 105, "max_ms": 4.0, "p95_ms": 3.734}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.773, "count": 105, "max_ms": 76.268, "p95_ms": 71.883}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.101, "count": 105, "max_ms": 82.118, "p95_ms": 78.178}, "yolo11_640_fire_11class": {"avg_ms": 69.401, "count": 105, "max_ms": 77.806, "p95_ms": 76.293}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.153, "count": 105, "max_ms": 86.492, "p95_ms": 84.834}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.193, "count": 105, "max_ms": 97.038, "p95_ms": 95.768}, "read_interval": {"avg_ms": 288.339, "count": 104, "max_ms": 19177.371, "p95_ms": 113.251}, "triton_wait": {"avg_ms": 77.458, "count": 105, "max_ms": 86.504, "p95_ms": 84.85}, "visualize": {"avg_ms": 3.772, "count": 105, "max_ms": 4.464, "p95_ms": 3.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1279}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4258.703, "limit_mb": 8796093022207.996}, "detail": {"pid": 27591, "restart_count": 192, "start_count": 193}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 167, "h264_decode_error_count": 223, "h264_decode_error_count_per_hour": 45, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:41:54+09:00", "last_error_summary": "[h264 @ 0x55bff17ab600] left block unavailable for requested intra mode [h264 @ 0x55bff17ab600] error while decoding MB 0 18, bytestream 339 [h264 @ 0x55bff237bf80] left block unavailable for requested intra mode [h264 @ 0x55bff237bf80] error while decoding MB 0 11, bytestream 2849 [h264 @ 0x55bff17ee500] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bff17ee500] error while decoding MB 0 15, bytestream 3070 [h264 @ 0x55bff18c3880] top block unavailable for requested intra mod", "last_exit_at": "2026-07-01T22:41:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:41:57+09:00", "pid": 27591, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.547, "name": "ffmpeg", "pid": 27591, "rss_mb": 136.547, "state": "S (sleeping)", "threads": 106}, "restart_count": 192, "restart_count_per_hour": 8, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 193}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.734, "sample_count": 2619, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 153557, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:41:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.875, "last_frame_pushed_at": "2026-07-01T22:41:54+09:00", "last_frame_read_age_sec": 4.939, "last_frame_read_at": "2026-07-01T22:41:54+09:00", "mjpeg": {"age_sec": 4.859, "exists": true, "mtime": "2026-07-01T22:41:54.358957+09:00", "mtime_epoch": 1782913314.3589573, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 105219}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:41:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 940.395, "name": "celery", "pid": 237, "rss_mb": 940.395, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.003, "pushed": 2.003, "read": 1.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.857, "count": 60, "max_ms": 2.06, "p95_ms": 2.052}, "gstreamer_push": {"avg_ms": 1.545, "count": 60, "max_ms": 4.0, "p95_ms": 3.757}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.671, "count": 60, "max_ms": 71.062, "p95_ms": 66.879}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.643, "count": 60, "max_ms": 79.613, "p95_ms": 77.795}, "yolo11_640_fire_11class": {"avg_ms": 67.137, "count": 60, "max_ms": 77.806, "p95_ms": 74.906}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.765, "count": 60, "max_ms": 86.237, "p95_ms": 84.722}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.673, "count": 60, "max_ms": 97.038, "p95_ms": 95.768}, "read_interval": {"avg_ms": 425.694, "count": 59, "max_ms": 19177.371, "p95_ms": 113.251}, "triton_wait": {"avg_ms": 74.857, "count": 60, "max_ms": 86.252, "p95_ms": 84.733}, "visualize": {"avg_ms": 3.783, "count": 60, "max_ms": 4.005, "p95_ms": 3.94}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2224}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4081.152, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555c60d29d00] error while decoding MB 5 32, bytestream -19 [in#0/rtsp @ 0x555c60cfde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555c60cfde00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:42:15+09:00", "last_error_summary": "[h264 @ 0x555c60d29d00] error while decoding MB 5 32, bytestream -19 [in#0/rtsp @ 0x555c60cfde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555c60cfde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:38:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 245.109, "last_started_at": "2026-07-01T22:38:10+09:00", "pid": 27439, "process": {"exists": false, "pid": 27439}, "restart_count": 29, "restart_count_per_hour": 10, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.85, "sample_count": 2226, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 266969, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:42:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-01T22:42:15+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T22:42:15+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-01T22:42:15.179332+09:00", "mtime_epoch": 1782913335.1793325, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 85704}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:42:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.328, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.693, "pushed": 0.693, "read": 0.751}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.395, "count": 12, "max_ms": 3.46, "p95_ms": 3.46}, "gstreamer_push": {"avg_ms": 1.745, "count": 12, "max_ms": 5.755, "p95_ms": 5.755}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.957, "count": 12, "max_ms": 76.33, "p95_ms": 76.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.751, "count": 12, "max_ms": 77.832, "p95_ms": 77.832}, "yolo11_640_fire_11class": {"avg_ms": 66.288, "count": 12, "max_ms": 76.342, "p95_ms": 76.342}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.548, "count": 12, "max_ms": 87.152, "p95_ms": 87.152}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.085, "count": 12, "max_ms": 99.662, "p95_ms": 99.662}, "read_interval": {"avg_ms": 2317.612, "count": 13, "max_ms": 16114.386, "p95_ms": 16114.386}, "triton_wait": {"avg_ms": 73.56, "count": 12, "max_ms": 87.163, "p95_ms": 87.163}, "visualize": {"avg_ms": 3.692, "count": 12, "max_ms": 3.878, "p95_ms": 3.878}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.314, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2224}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4080.93, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:42:15+09:00", "last_error_summary": "[h264 @ 0x555c60d29d00] error while decoding MB 5 32, bytestream -19 [in#0/rtsp @ 0x555c60cfde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555c60cfde00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:42:15+09:00", "last_error_summary": "[h264 @ 0x555c60d29d00] error while decoding MB 5 32, bytestream -19 [in#0/rtsp @ 0x555c60cfde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555c60cfde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:42:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 245.139, "last_started_at": "2026-07-01T22:38:10+09:00", "pid": 27439, "process": {"exists": false, "pid": 27439}, "restart_count": 29, "restart_count_per_hour": 10, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.85, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 266970, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:42:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T22:42:15+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-01T22:42:15+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-01T22:42:15.179332+09:00", "mtime_epoch": 1782913335.1793325, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 85704}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:42:15+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.328, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.75, "pushed": 0.75, "read": 0.75}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.482, "count": 13, "max_ms": 4.526, "p95_ms": 4.526}, "gstreamer_push": {"avg_ms": 1.714, "count": 13, "max_ms": 5.755, "p95_ms": 5.755}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.806, "count": 13, "max_ms": 76.33, "p95_ms": 76.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.768, "count": 13, "max_ms": 77.832, "p95_ms": 77.832}, "yolo11_640_fire_11class": {"avg_ms": 66.419, "count": 13, "max_ms": 76.342, "p95_ms": 76.342}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.493, "count": 13, "max_ms": 87.152, "p95_ms": 87.152}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.107, "count": 13, "max_ms": 99.662, "p95_ms": 99.662}, "read_interval": {"avg_ms": 2317.612, "count": 13, "max_ms": 16114.386, "p95_ms": 16114.386}, "triton_wait": {"avg_ms": 73.505, "count": 13, "max_ms": 87.163, "p95_ms": 87.163}, "visualize": {"avg_ms": 3.689, "count": 13, "max_ms": 3.878, "p95_ms": 3.878}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.344, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2224}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4080.613, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:42:15+09:00", "last_error_summary": "[h264 @ 0x555c60d29d00] error while decoding MB 5 32, bytestream -19 [in#0/rtsp @ 0x555c60cfde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555c60cfde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:42:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 245.146, "last_started_at": "2026-07-01T22:38:10+09:00", "pid": 27439, "process": {"exists": false, "pid": 27439}, "restart_count": 29, "restart_count_per_hour": 10, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.85, "sample_count": 2226, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 266970, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:42:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T22:42:15+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-01T22:42:15+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-01T22:42:15.179332+09:00", "mtime_epoch": 1782913335.1793325, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 85704}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:42:15+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.328, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.749, "pushed": 0.749, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.482, "count": 13, "max_ms": 4.526, "p95_ms": 4.526}, "gstreamer_push": {"avg_ms": 1.714, "count": 13, "max_ms": 5.755, "p95_ms": 5.755}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.806, "count": 13, "max_ms": 76.33, "p95_ms": 76.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.768, "count": 13, "max_ms": 77.832, "p95_ms": 77.832}, "yolo11_640_fire_11class": {"avg_ms": 66.419, "count": 13, "max_ms": 76.342, "p95_ms": 76.342}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.493, "count": 13, "max_ms": 87.152, "p95_ms": 87.152}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.107, "count": 13, "max_ms": 99.662, "p95_ms": 99.662}, "read_interval": {"avg_ms": 2317.612, "count": 13, "max_ms": 16114.386, "p95_ms": 16114.386}, "triton_wait": {"avg_ms": 73.505, "count": 13, "max_ms": 87.163, "p95_ms": 87.163}, "visualize": {"avg_ms": 3.689, "count": 13, "max_ms": 3.878, "p95_ms": 3.878}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.351, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2224}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4100.938, "limit_mb": 8796093022207.996}, "detail": {"pid": 27698, "restart_count": 30, "start_count": 31}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:42:15+09:00", "last_error_summary": "[h264 @ 0x555c60d29d00] error while decoding MB 5 32, bytestream -19 [in#0/rtsp @ 0x555c60cfde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555c60cfde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:42:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:42:17+09:00", "pid": 27698, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.316, "name": "ffmpeg", "pid": 27698, "rss_mb": 57.316, "state": "S (sleeping)", "threads": 1}, "restart_count": 30, "restart_count_per_hour": 11, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.846, "sample_count": 2188, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 266970, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:42:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.263, "last_frame_pushed_at": "2026-07-01T22:42:15+09:00", "last_frame_read_age_sec": 4.352, "last_frame_read_at": "2026-07-01T22:42:15+09:00", "mjpeg": {"age_sec": 4.246, "exists": true, "mtime": "2026-07-01T22:42:15.283334+09:00", "mtime_epoch": 1782913335.2833345, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86361}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:42:19+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 991.328, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.602, "pushed": 0.602, "read": 0.602}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.482, "count": 13, "max_ms": 4.526, "p95_ms": 4.526}, "gstreamer_push": {"avg_ms": 1.714, "count": 13, "max_ms": 5.755, "p95_ms": 5.755}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.806, "count": 13, "max_ms": 76.33, "p95_ms": 76.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.768, "count": 13, "max_ms": 77.832, "p95_ms": 77.832}, "yolo11_640_fire_11class": {"avg_ms": 66.419, "count": 13, "max_ms": 76.342, "p95_ms": 76.342}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.493, "count": 13, "max_ms": 87.152, "p95_ms": 87.152}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.107, "count": 13, "max_ms": 99.662, "p95_ms": 99.662}, "read_interval": {"avg_ms": 2317.612, "count": 13, "max_ms": 16114.386, "p95_ms": 16114.386}, "triton_wait": {"avg_ms": 73.505, "count": 13, "max_ms": 87.163, "p95_ms": 87.163}, "visualize": {"avg_ms": 3.689, "count": 13, "max_ms": 3.878, "p95_ms": 3.878}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.602, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4111.945, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a445da9d40] error while decoding MB 99 31, bytestream -5 [in#0/rtsp @ 0x55a445d69e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a445d69e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:46:12+09:00", "last_error_summary": "[h264 @ 0x55a445da9d40] error while decoding MB 99 31, bytestream -5 [in#0/rtsp @ 0x55a445d69e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a445d69e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:42:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 235.313, "last_started_at": "2026-07-01T22:42:17+09:00", "pid": 27698, "process": {"exists": false, "pid": 27698}, "restart_count": 30, "restart_count_per_hour": 11, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.63, "sample_count": 2251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 268872, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:46:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-01T22:46:12+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-01T22:46:12+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-01T22:46:12.783588+09:00", "mtime_epoch": 1782913572.7835884, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 84167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:46:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 997.516, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 58, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.939, "pushed": 1.939, "read": 1.939}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.425, "count": 58, "max_ms": 3.724, "p95_ms": 3.537}, "gstreamer_push": {"avg_ms": 1.412, "count": 58, "max_ms": 3.905, "p95_ms": 1.522}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.691, "count": 58, "max_ms": 76.441, "p95_ms": 73.627}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.84, "count": 58, "max_ms": 80.564, "p95_ms": 76.84}, "yolo11_640_fire_11class": {"avg_ms": 67.171, "count": 58, "max_ms": 75.861, "p95_ms": 75.329}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.664, "count": 58, "max_ms": 87.756, "p95_ms": 85.634}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.237, "count": 58, "max_ms": 100.116, "p95_ms": 98.027}, "read_interval": {"avg_ms": 516.263, "count": 58, "max_ms": 15088.339, "p95_ms": 117.123}, "triton_wait": {"avg_ms": 74.835, "count": 58, "max_ms": 87.768, "p95_ms": 85.647}, "visualize": {"avg_ms": 3.676, "count": 58, "max_ms": 4.028, "p95_ms": 3.869}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4111.855, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:46:12+09:00", "last_error_summary": "[h264 @ 0x55a445da9d40] error while decoding MB 99 31, bytestream -5 [in#0/rtsp @ 0x55a445d69e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a445d69e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:46:12+09:00", "last_error_summary": "[h264 @ 0x55a445da9d40] error while decoding MB 99 31, bytestream -5 [in#0/rtsp @ 0x55a445d69e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a445d69e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:46:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 235.333, "last_started_at": "2026-07-01T22:42:17+09:00", "pid": 27698, "process": {"exists": false, "pid": 27698}, "restart_count": 30, "restart_count_per_hour": 11, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.626, "sample_count": 2252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 268873, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:46:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T22:46:12+09:00", "last_frame_read_age_sec": 0.081, "last_frame_read_at": "2026-07-01T22:46:12+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-01T22:46:12.783588+09:00", "mtime_epoch": 1782913572.7835884, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 84167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:46:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 997.516, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.971, "pushed": 1.971, "read": 1.938}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.438, "count": 59, "max_ms": 4.162, "p95_ms": 3.537}, "gstreamer_push": {"avg_ms": 1.409, "count": 59, "max_ms": 3.905, "p95_ms": 1.522}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.406, "count": 59, "max_ms": 76.441, "p95_ms": 73.627}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.615, "count": 59, "max_ms": 80.564, "p95_ms": 76.84}, "yolo11_640_fire_11class": {"avg_ms": 66.956, "count": 59, "max_ms": 75.861, "p95_ms": 75.329}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.351, "count": 59, "max_ms": 87.756, "p95_ms": 85.634}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.005, "count": 59, "max_ms": 100.116, "p95_ms": 98.027}, "read_interval": {"avg_ms": 516.263, "count": 58, "max_ms": 15088.339, "p95_ms": 117.123}, "triton_wait": {"avg_ms": 74.519, "count": 59, "max_ms": 87.768, "p95_ms": 85.647}, "visualize": {"avg_ms": 3.723, "count": 59, "max_ms": 6.437, "p95_ms": 4.021}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4111.824, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:46:12+09:00", "last_error_summary": "[h264 @ 0x55a445da9d40] error while decoding MB 99 31, bytestream -5 [in#0/rtsp @ 0x55a445d69e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a445d69e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:46:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 235.341, "last_started_at": "2026-07-01T22:42:17+09:00", "pid": 27698, "process": {"exists": false, "pid": 27698}, "restart_count": 30, "restart_count_per_hour": 11, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.626, "sample_count": 2252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 268873, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:46:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T22:46:12+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-01T22:46:12+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-01T22:46:12.783588+09:00", "mtime_epoch": 1782913572.7835884, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 84167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:46:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 997.516, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.97, "pushed": 1.97, "read": 1.937}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.438, "count": 59, "max_ms": 4.162, "p95_ms": 3.537}, "gstreamer_push": {"avg_ms": 1.409, "count": 59, "max_ms": 3.905, "p95_ms": 1.522}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.406, "count": 59, "max_ms": 76.441, "p95_ms": 73.627}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.615, "count": 59, "max_ms": 80.564, "p95_ms": 76.84}, "yolo11_640_fire_11class": {"avg_ms": 66.956, "count": 59, "max_ms": 75.861, "p95_ms": 75.329}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.351, "count": 59, "max_ms": 87.756, "p95_ms": 85.634}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.005, "count": 59, "max_ms": 100.116, "p95_ms": 98.027}, "read_interval": {"avg_ms": 516.263, "count": 58, "max_ms": 15088.339, "p95_ms": 117.123}, "triton_wait": {"avg_ms": 74.519, "count": 59, "max_ms": 87.768, "p95_ms": 85.647}, "visualize": {"avg_ms": 3.723, "count": 59, "max_ms": 6.437, "p95_ms": 4.021}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2240}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4231.91, "limit_mb": 8796093022207.996}, "detail": {"pid": 27850, "restart_count": 31, "start_count": 32}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:46:12+09:00", "last_error_summary": "[h264 @ 0x55a445da9d40] error while decoding MB 99 31, bytestream -5 [in#0/rtsp @ 0x55a445d69e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a445d69e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:46:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:46:15+09:00", "pid": 27850, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.84, "name": "ffmpeg", "pid": 27850, "rss_mb": 156.676, "state": "R (running)", "threads": 49}, "restart_count": 31, "restart_count_per_hour": 12, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.626, "sample_count": 2207, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 268873, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:46:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.99, "last_frame_pushed_at": "2026-07-01T22:46:12+09:00", "last_frame_read_age_sec": 5.065, "last_frame_read_at": "2026-07-01T22:46:12+09:00", "mjpeg": {"age_sec": 4.974, "exists": true, "mtime": "2026-07-01T22:46:12.871590+09:00", "mtime_epoch": 1782913572.87159, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 84857}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:46:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 997.516, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.663, "pushed": 0.663, "read": 0.663}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.478, "count": 14, "max_ms": 4.162, "p95_ms": 4.162}, "gstreamer_push": {"avg_ms": 1.499, "count": 14, "max_ms": 3.648, "p95_ms": 3.648}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.049, "count": 14, "max_ms": 57.073, "p95_ms": 57.073}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.735, "count": 14, "max_ms": 58.891, "p95_ms": 58.891}, "yolo11_640_fire_11class": {"avg_ms": 51.966, "count": 14, "max_ms": 56.03, "p95_ms": 56.03}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.269, "count": 14, "max_ms": 58.512, "p95_ms": 58.512}}, "truncated_count": 0}, "processing_total": {"avg_ms": 66.612, "count": 14, "max_ms": 73.518, "p95_ms": 73.518}, "read_interval": {"avg_ms": 1791.64, "count": 14, "max_ms": 15088.339, "p95_ms": 15088.339}, "triton_wait": {"avg_ms": 53.939, "count": 14, "max_ms": 58.903, "p95_ms": 58.903}, "visualize": {"avg_ms": 3.855, "count": 14, "max_ms": 6.437, "p95_ms": 6.437}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.116, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4113.84, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f9451834c0] error while decoding MB 77 51, bytestream -5 [in#0/rtsp @ 0x55f944997e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f944997e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:49:00+09:00", "last_error_summary": "[h264 @ 0x55f9451834c0] error while decoding MB 77 51, bytestream -5 [in#0/rtsp @ 0x55f944997e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f944997e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:46:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.443, "last_started_at": "2026-07-01T22:46:15+09:00", "pid": 27850, "process": {"exists": false, "pid": 27850}, "restart_count": 31, "restart_count_per_hour": 12, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.37, "sample_count": 2171, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270050, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:49:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-01T22:49:00+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T22:49:00+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-01T22:49:00.230565+09:00", "mtime_epoch": 1782913740.2305646, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91898}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:49:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 995.395, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.693, "pushed": 0.693, "read": 0.75}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.483, "count": 12, "max_ms": 3.963, "p95_ms": 3.963}, "gstreamer_push": {"avg_ms": 1.361, "count": 12, "max_ms": 1.506, "p95_ms": 1.506}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.865, "count": 12, "max_ms": 53.231, "p95_ms": 53.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.568, "count": 12, "max_ms": 57.384, "p95_ms": 57.384}, "yolo11_640_fire_11class": {"avg_ms": 50.934, "count": 12, "max_ms": 56.655, "p95_ms": 56.655}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.609, "count": 12, "max_ms": 56.888, "p95_ms": 56.888}}, "truncated_count": 0}, "processing_total": {"avg_ms": 64.7, "count": 12, "max_ms": 69.441, "p95_ms": 69.441}, "read_interval": {"avg_ms": 2691.185, "count": 13, "max_ms": 17722.151, "p95_ms": 17722.151}, "triton_wait": {"avg_ms": 52.255, "count": 12, "max_ms": 57.394, "p95_ms": 57.394}, "visualize": {"avg_ms": 3.723, "count": 12, "max_ms": 4.167, "p95_ms": 4.167}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.327, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4113.469, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:49:00+09:00", "last_error_summary": "[h264 @ 0x55f9451834c0] error while decoding MB 77 51, bytestream -5 [in#0/rtsp @ 0x55f944997e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f944997e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:49:00+09:00", "last_error_summary": "[h264 @ 0x55f9451834c0] error while decoding MB 77 51, bytestream -5 [in#0/rtsp @ 0x55f944997e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f944997e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:49:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.462, "last_started_at": "2026-07-01T22:46:15+09:00", "pid": 27850, "process": {"exists": false, "pid": 27850}, "restart_count": 31, "restart_count_per_hour": 12, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.346, "sample_count": 2172, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270051, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:49:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T22:49:00+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-01T22:49:00+09:00", "mjpeg": {"age_sec": 0.075, "exists": true, "mtime": "2026-07-01T22:49:00.230565+09:00", "mtime_epoch": 1782913740.2305646, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91898}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:49:00+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 995.395, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.749, "pushed": 0.749, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.478, "count": 13, "max_ms": 3.963, "p95_ms": 3.963}, "gstreamer_push": {"avg_ms": 1.366, "count": 13, "max_ms": 1.506, "p95_ms": 1.506}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.984, "count": 13, "max_ms": 56.409, "p95_ms": 56.409}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.941, "count": 13, "max_ms": 57.384, "p95_ms": 57.384}, "yolo11_640_fire_11class": {"avg_ms": 51.357, "count": 13, "max_ms": 56.655, "p95_ms": 56.655}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.028, "count": 13, "max_ms": 57.062, "p95_ms": 57.062}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.447, "count": 13, "max_ms": 74.409, "p95_ms": 74.409}, "read_interval": {"avg_ms": 2691.185, "count": 13, "max_ms": 17722.151, "p95_ms": 17722.151}, "triton_wait": {"avg_ms": 52.626, "count": 13, "max_ms": 57.394, "p95_ms": 57.394}, "visualize": {"avg_ms": 3.852, "count": 13, "max_ms": 5.396, "p95_ms": 5.396}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.346, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4113.332, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:49:00+09:00", "last_error_summary": "[h264 @ 0x55f9451834c0] error while decoding MB 77 51, bytestream -5 [in#0/rtsp @ 0x55f944997e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f944997e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:49:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.469, "last_started_at": "2026-07-01T22:46:15+09:00", "pid": 27850, "process": {"exists": false, "pid": 27850}, "restart_count": 31, "restart_count_per_hour": 12, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.346, "sample_count": 2172, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270051, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:49:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T22:49:00+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-01T22:49:00+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-01T22:49:00.230565+09:00", "mtime_epoch": 1782913740.2305646, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91898}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:49:00+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 995.395, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.749, "pushed": 0.749, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.478, "count": 13, "max_ms": 3.963, "p95_ms": 3.963}, "gstreamer_push": {"avg_ms": 1.366, "count": 13, "max_ms": 1.506, "p95_ms": 1.506}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.984, "count": 13, "max_ms": 56.409, "p95_ms": 56.409}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.941, "count": 13, "max_ms": 57.384, "p95_ms": 57.384}, "yolo11_640_fire_11class": {"avg_ms": 51.357, "count": 13, "max_ms": 56.655, "p95_ms": 56.655}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.028, "count": 13, "max_ms": 57.062, "p95_ms": 57.062}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.447, "count": 13, "max_ms": 74.409, "p95_ms": 74.409}, "read_interval": {"avg_ms": 2691.185, "count": 13, "max_ms": 17722.151, "p95_ms": 17722.151}, "triton_wait": {"avg_ms": 52.626, "count": 13, "max_ms": 57.394, "p95_ms": 57.394}, "visualize": {"avg_ms": 3.852, "count": 13, "max_ms": 5.396, "p95_ms": 5.396}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.353, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2250}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4178.773, "limit_mb": 8796093022207.996}, "detail": {"pid": 28002, "restart_count": 32, "start_count": 33}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:49:00+09:00", "last_error_summary": "[h264 @ 0x55f9451834c0] error while decoding MB 77 51, bytestream -5 [in#0/rtsp @ 0x55f944997e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f944997e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:49:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:49:02+09:00", "pid": 28002, "process": {"exists": true, "fd_count": 5, "hwm_mb": 95.254, "name": "ffmpeg", "pid": 28002, "rss_mb": 95.254, "state": "S (sleeping)", "threads": 19}, "restart_count": 32, "restart_count_per_hour": 13, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.346, "sample_count": 2133, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270051, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:49:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.35, "last_frame_pushed_at": "2026-07-01T22:49:00+09:00", "last_frame_read_age_sec": 4.426, "last_frame_read_at": "2026-07-01T22:49:00+09:00", "mjpeg": {"age_sec": 4.336, "exists": true, "mtime": "2026-07-01T22:49:00.314566+09:00", "mtime_epoch": 1782913740.314566, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:49:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 995.395, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.599, "pushed": 0.599, "read": 0.599}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.478, "count": 13, "max_ms": 3.963, "p95_ms": 3.963}, "gstreamer_push": {"avg_ms": 1.366, "count": 13, "max_ms": 1.506, "p95_ms": 1.506}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.984, "count": 13, "max_ms": 56.409, "p95_ms": 56.409}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.941, "count": 13, "max_ms": 57.384, "p95_ms": 57.384}, "yolo11_640_fire_11class": {"avg_ms": 51.357, "count": 13, "max_ms": 56.655, "p95_ms": 56.655}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.028, "count": 13, "max_ms": 57.062, "p95_ms": 57.062}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.447, "count": 13, "max_ms": 74.409, "p95_ms": 74.409}, "read_interval": {"avg_ms": 2691.185, "count": 13, "max_ms": 17722.151, "p95_ms": 17722.151}, "triton_wait": {"avg_ms": 52.626, "count": 13, "max_ms": 57.394, "p95_ms": 57.394}, "visualize": {"avg_ms": 3.852, "count": 13, "max_ms": 5.396, "p95_ms": 5.396}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.69, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2284}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4135.23, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b159ed6900] error while decoding MB 68 35, bytestream -17 [NULL @ 0x55b1593b4200] missing picture in access unit with size 18 [h264 @ 0x55b159b9af00] no frame! [in#0/rtsp @ 0x55b1593afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b1593afe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b1593b4840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:57:01+09:00", "last_error_summary": "[h264 @ 0x55b159ed6900] error while decoding MB 68 35, bytestream -17 [NULL @ 0x55b1593b4200] missing picture in access unit with size 18 [h264 @ 0x55b159b9af00] no frame! [in#0/rtsp @ 0x55b1593afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b1593afe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b1593b4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:49:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 479.288, "last_started_at": "2026-07-01T22:49:02+09:00", "pid": 28002, "process": {"exists": false, "pid": 28002}, "restart_count": 32, "restart_count_per_hour": 12, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.426, "sample_count": 2515, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 274162, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:57:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T22:57:01+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-01T22:57:01+09:00", "mjpeg": {"age_sec": 0.068, "exists": true, "mtime": "2026-07-01T22:57:01.870273+09:00", "mtime_epoch": 1782914221.8702726, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 85578}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:57:01+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 997.457, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 103, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.434, "pushed": 3.434, "read": 3.434}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.346, "count": 103, "max_ms": 3.736, "p95_ms": 3.516}, "gstreamer_push": {"avg_ms": 1.528, "count": 103, "max_ms": 6.023, "p95_ms": 3.638}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.128, "count": 103, "max_ms": 78.493, "p95_ms": 74.616}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.315, "count": 103, "max_ms": 83.947, "p95_ms": 79.948}, "yolo11_640_fire_11class": {"avg_ms": 69.436, "count": 103, "max_ms": 79.991, "p95_ms": 77.238}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.831, "count": 103, "max_ms": 88.22, "p95_ms": 86.838}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.884, "count": 103, "max_ms": 100.194, "p95_ms": 98.04}, "read_interval": {"avg_ms": 291.634, "count": 103, "max_ms": 19068.075, "p95_ms": 116.331}, "triton_wait": {"avg_ms": 78.913, "count": 103, "max_ms": 88.232, "p95_ms": 86.852}, "visualize": {"avg_ms": 3.583, "count": 103, "max_ms": 4.254, "p95_ms": 3.772}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2284}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4135.262, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:57:01+09:00", "last_error_summary": "[h264 @ 0x55b159ed6900] error while decoding MB 68 35, bytestream -17 [NULL @ 0x55b1593b4200] missing picture in access unit with size 18 [h264 @ 0x55b159b9af00] no frame! [in#0/rtsp @ 0x55b1593afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b1593afe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b1593b4840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:57:01+09:00", "last_error_summary": "[h264 @ 0x55b159ed6900] error while decoding MB 68 35, bytestream -17 [NULL @ 0x55b1593b4200] missing picture in access unit with size 18 [h264 @ 0x55b159b9af00] no frame! [in#0/rtsp @ 0x55b1593afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b1593afe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b1593b4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:57:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 479.301, "last_started_at": "2026-07-01T22:49:02+09:00", "pid": 28002, "process": {"exists": false, "pid": 28002}, "restart_count": 32, "restart_count_per_hour": 12, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.426, "sample_count": 2515, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 274162, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:57:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-01T22:57:01+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-01T22:57:01+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-01T22:57:01.870273+09:00", "mtime_epoch": 1782914221.8702726, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 85578}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:57:01+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 997.457, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.444, "pushed": 3.444, "read": 3.41}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.346, "count": 103, "max_ms": 3.736, "p95_ms": 3.516}, "gstreamer_push": {"avg_ms": 1.528, "count": 103, "max_ms": 6.023, "p95_ms": 3.638}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.128, "count": 103, "max_ms": 78.493, "p95_ms": 74.616}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.315, "count": 103, "max_ms": 83.947, "p95_ms": 79.948}, "yolo11_640_fire_11class": {"avg_ms": 69.436, "count": 103, "max_ms": 79.991, "p95_ms": 77.238}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.831, "count": 103, "max_ms": 88.22, "p95_ms": 86.838}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.884, "count": 103, "max_ms": 100.194, "p95_ms": 98.04}, "read_interval": {"avg_ms": 293.388, "count": 102, "max_ms": 19068.075, "p95_ms": 116.331}, "triton_wait": {"avg_ms": 78.913, "count": 103, "max_ms": 88.232, "p95_ms": 86.852}, "visualize": {"avg_ms": 3.583, "count": 103, "max_ms": 4.254, "p95_ms": 3.772}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2284}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4135.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:57:01+09:00", "last_error_summary": "[h264 @ 0x55b159ed6900] error while decoding MB 68 35, bytestream -17 [NULL @ 0x55b1593b4200] missing picture in access unit with size 18 [h264 @ 0x55b159b9af00] no frame! [in#0/rtsp @ 0x55b1593afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b1593afe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b1593b4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:57:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 479.31, "last_started_at": "2026-07-01T22:49:02+09:00", "pid": 28002, "process": {"exists": false, "pid": 28002}, "restart_count": 32, "restart_count_per_hour": 12, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.426, "sample_count": 2515, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 274162, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:57:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.026, "last_frame_pushed_at": "2026-07-01T22:57:01+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-01T22:57:01+09:00", "mjpeg": {"age_sec": 0.009, "exists": true, "mtime": "2026-07-01T22:57:01.950274+09:00", "mtime_epoch": 1782914221.950274, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86605}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:57:01+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 997.457, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.443, "pushed": 3.443, "read": 3.409}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.346, "count": 103, "max_ms": 3.736, "p95_ms": 3.516}, "gstreamer_push": {"avg_ms": 1.528, "count": 103, "max_ms": 6.023, "p95_ms": 3.638}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.128, "count": 103, "max_ms": 78.493, "p95_ms": 74.616}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.315, "count": 103, "max_ms": 83.947, "p95_ms": 79.948}, "yolo11_640_fire_11class": {"avg_ms": 69.436, "count": 103, "max_ms": 79.991, "p95_ms": 77.238}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.831, "count": 103, "max_ms": 88.22, "p95_ms": 86.838}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.884, "count": 103, "max_ms": 100.194, "p95_ms": 98.04}, "read_interval": {"avg_ms": 293.388, "count": 102, "max_ms": 19068.075, "p95_ms": 116.331}, "triton_wait": {"avg_ms": 78.913, "count": 103, "max_ms": 88.232, "p95_ms": 86.852}, "visualize": {"avg_ms": 3.583, "count": 103, "max_ms": 4.254, "p95_ms": 3.772}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2284}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4152.824, "limit_mb": 8796093022207.996}, "detail": {"pid": 28154, "restart_count": 33, "start_count": 34}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:57:01+09:00", "last_error_summary": "[h264 @ 0x55b159ed6900] error while decoding MB 68 35, bytestream -17 [NULL @ 0x55b1593b4200] missing picture in access unit with size 18 [h264 @ 0x55b159b9af00] no frame! [in#0/rtsp @ 0x55b1593afe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b1593afe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55b1593b4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T22:57:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:57:04+09:00", "pid": 28154, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.906, "name": "ffmpeg", "pid": 28154, "rss_mb": 57.906, "state": "S (sleeping)", "threads": 1}, "restart_count": 33, "restart_count_per_hour": 13, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.476, "sample_count": 2475, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 274162, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:57:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.487, "last_frame_pushed_at": "2026-07-01T22:57:01+09:00", "last_frame_read_age_sec": 4.552, "last_frame_read_at": "2026-07-01T22:57:01+09:00", "mjpeg": {"age_sec": 4.47, "exists": true, "mtime": "2026-07-01T22:57:01.950274+09:00", "mtime_epoch": 1782914221.950274, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86605}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T22:57:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 997.457, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.104, "pushed": 2.104, "read": 2.07}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.39, "count": 63, "max_ms": 3.573, "p95_ms": 3.516}, "gstreamer_push": {"avg_ms": 1.569, "count": 63, "max_ms": 6.023, "p95_ms": 3.67}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.187, "count": 63, "max_ms": 75.228, "p95_ms": 70.348}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.494, "count": 63, "max_ms": 83.947, "p95_ms": 78.477}, "yolo11_640_fire_11class": {"avg_ms": 66.754, "count": 63, "max_ms": 76.729, "p95_ms": 74.716}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.367, "count": 63, "max_ms": 87.519, "p95_ms": 84.948}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.522, "count": 63, "max_ms": 99.896, "p95_ms": 96.987}, "read_interval": {"avg_ms": 411.064, "count": 62, "max_ms": 19068.075, "p95_ms": 116.045}, "triton_wait": {"avg_ms": 76.462, "count": 63, "max_ms": 87.532, "p95_ms": 84.961}, "visualize": {"avg_ms": 3.619, "count": 63, "max_ms": 4.254, "p95_ms": 3.738}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1357}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4149.18, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5619e4871340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e4871340] error while decoding MB 0 38, bytestream 7413 [h264 @ 0x5619e479b340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e479b340] error while decoding MB 0 5, bytestream 4260 [h264 @ 0x5619e479b340] cabac decode of qscale diff failed at 46 26 [h264 @ 0x5619e479b340] error while decoding MB 46 26, bytestream 2459 [h264 @ 0x5619e4730480] error while decoding MB 63 27, bytestream ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 168, "h264_decode_error_count": 229, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:58:27+09:00", "last_error_summary": "[h264 @ 0x5619e4871340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e4871340] error while decoding MB 0 38, bytestream 7413 [h264 @ 0x5619e479b340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e479b340] error while decoding MB 0 5, bytestream 4260 [h264 @ 0x5619e479b340] cabac decode of qscale diff failed at 46 26 [h264 @ 0x5619e479b340] error while decoding MB 46 26, bytestream 2459 [h264 @ 0x5619e4730480] error while decoding MB 63 27, bytestream ", "last_exit_at": "2026-07-01T22:41:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 990.251, "last_started_at": "2026-07-01T22:41:57+09:00", "pid": 27591, "process": {"exists": false, "pid": 27591}, "restart_count": 192, "restart_count_per_hour": 7, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 193}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.605, "sample_count": 2649, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 162948, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:58:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-01T22:58:27+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-01T22:58:27+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-01T22:58:27.411556+09:00", "mtime_epoch": 1782914307.411556, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:58:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 943.938, "name": "celery", "pid": 237, "rss_mb": 936.918, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.206, "pushed": 3.206, "read": 3.206}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.527, "count": 96, "max_ms": 2.045, "p95_ms": 1.585}, "gstreamer_push": {"avg_ms": 1.691, "count": 96, "max_ms": 6.222, "p95_ms": 4.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.607, "count": 96, "max_ms": 78.132, "p95_ms": 75.217}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.265, "count": 96, "max_ms": 84.542, "p95_ms": 80.395}, "yolo11_640_fire_11class": {"avg_ms": 71.856, "count": 96, "max_ms": 85.399, "p95_ms": 78.173}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.464, "count": 96, "max_ms": 91.57, "p95_ms": 86.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.979, "count": 96, "max_ms": 102.009, "p95_ms": 96.969}, "read_interval": {"avg_ms": 312.317, "count": 96, "max_ms": 19856.24, "p95_ms": 117.378}, "triton_wait": {"avg_ms": 79.493, "count": 96, "max_ms": 91.582, "p95_ms": 86.286}, "visualize": {"avg_ms": 3.674, "count": 96, "max_ms": 4.104, "p95_ms": 3.845}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1357}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4148.953, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T22:58:27+09:00", "last_error_summary": "[h264 @ 0x5619e4871340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e4871340] error while decoding MB 0 38, bytestream 7413 [h264 @ 0x5619e479b340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e479b340] error while decoding MB 0 5, bytestream 4260 [h264 @ 0x5619e479b340] cabac decode of qscale diff failed at 46 26 [h264 @ 0x5619e479b340] error while decoding MB 46 26, bytestream 2459 [h264 @ 0x5619e4730480] error while decoding MB 63 27, bytestream ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 168, "h264_decode_error_count": 229, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:58:27+09:00", "last_error_summary": "[h264 @ 0x5619e4871340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e4871340] error while decoding MB 0 38, bytestream 7413 [h264 @ 0x5619e479b340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e479b340] error while decoding MB 0 5, bytestream 4260 [h264 @ 0x5619e479b340] cabac decode of qscale diff failed at 46 26 [h264 @ 0x5619e479b340] error while decoding MB 46 26, bytestream 2459 [h264 @ 0x5619e4730480] error while decoding MB 63 27, bytestream ", "last_exit_at": "2026-07-01T22:58:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 990.277, "last_started_at": "2026-07-01T22:41:57+09:00", "pid": 27591, "process": {"exists": false, "pid": 27591}, "restart_count": 192, "restart_count_per_hour": 7, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 193}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.605, "sample_count": 2649, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 162949, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:58:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T22:58:27+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-01T22:58:27+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-01T22:58:27.411556+09:00", "mtime_epoch": 1782914307.411556, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:58:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 943.938, "name": "celery", "pid": 237, "rss_mb": 936.918, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.236, "pushed": 3.236, "read": 3.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.529, "count": 97, "max_ms": 2.045, "p95_ms": 1.589}, "gstreamer_push": {"avg_ms": 1.688, "count": 97, "max_ms": 6.222, "p95_ms": 4.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.6, "count": 97, "max_ms": 78.132, "p95_ms": 75.217}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.2, "count": 97, "max_ms": 84.542, "p95_ms": 80.395}, "yolo11_640_fire_11class": {"avg_ms": 71.734, "count": 97, "max_ms": 85.399, "p95_ms": 78.173}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.341, "count": 97, "max_ms": 91.57, "p95_ms": 86.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.866, "count": 97, "max_ms": 102.009, "p95_ms": 96.969}, "read_interval": {"avg_ms": 312.317, "count": 96, "max_ms": 19856.24, "p95_ms": 117.378}, "triton_wait": {"avg_ms": 79.37, "count": 97, "max_ms": 91.582, "p95_ms": 86.286}, "visualize": {"avg_ms": 3.682, "count": 97, "max_ms": 4.466, "p95_ms": 3.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1357}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4148.746, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 168, "h264_decode_error_count": 229, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:58:27+09:00", "last_error_summary": "[h264 @ 0x5619e4871340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e4871340] error while decoding MB 0 38, bytestream 7413 [h264 @ 0x5619e479b340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e479b340] error while decoding MB 0 5, bytestream 4260 [h264 @ 0x5619e479b340] cabac decode of qscale diff failed at 46 26 [h264 @ 0x5619e479b340] error while decoding MB 46 26, bytestream 2459 [h264 @ 0x5619e4730480] error while decoding MB 63 27, bytestream ", "last_exit_at": "2026-07-01T22:58:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 990.285, "last_started_at": "2026-07-01T22:41:57+09:00", "pid": 27591, "process": {"exists": false, "pid": 27591}, "restart_count": 192, "restart_count_per_hour": 7, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 193}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.605, "sample_count": 2649, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 162949, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:58:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T22:58:27+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-01T22:58:27+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T22:58:27.411556+09:00", "mtime_epoch": 1782914307.411556, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125680}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:58:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 943.938, "name": "celery", "pid": 237, "rss_mb": 936.918, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.235, "pushed": 3.235, "read": 3.202}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.529, "count": 97, "max_ms": 2.045, "p95_ms": 1.589}, "gstreamer_push": {"avg_ms": 1.688, "count": 97, "max_ms": 6.222, "p95_ms": 4.225}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.6, "count": 97, "max_ms": 78.132, "p95_ms": 75.217}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.2, "count": 97, "max_ms": 84.542, "p95_ms": 80.395}, "yolo11_640_fire_11class": {"avg_ms": 71.734, "count": 97, "max_ms": 85.399, "p95_ms": 78.173}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.341, "count": 97, "max_ms": 91.57, "p95_ms": 86.273}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.866, "count": 97, "max_ms": 102.009, "p95_ms": 96.969}, "read_interval": {"avg_ms": 312.317, "count": 96, "max_ms": 19856.24, "p95_ms": 117.378}, "triton_wait": {"avg_ms": 79.37, "count": 97, "max_ms": 91.582, "p95_ms": 86.286}, "visualize": {"avg_ms": 3.682, "count": 97, "max_ms": 4.466, "p95_ms": 3.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1357}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4256.203, "limit_mb": 8796093022207.996}, "detail": {"pid": 28306, "restart_count": 193, "start_count": 194}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 168, "h264_decode_error_count": 229, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T22:58:27+09:00", "last_error_summary": "[h264 @ 0x5619e4871340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e4871340] error while decoding MB 0 38, bytestream 7413 [h264 @ 0x5619e479b340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5619e479b340] error while decoding MB 0 5, bytestream 4260 [h264 @ 0x5619e479b340] cabac decode of qscale diff failed at 46 26 [h264 @ 0x5619e479b340] error while decoding MB 46 26, bytestream 2459 [h264 @ 0x5619e4730480] error while decoding MB 63 27, bytestream ", "last_exit_at": "2026-07-01T22:58:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T22:58:30+09:00", "pid": 28306, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.734, "name": "ffmpeg", "pid": 28306, "rss_mb": 138.734, "state": "S (sleeping)", "threads": 106}, "restart_count": 193, "restart_count_per_hour": 8, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 194}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.591, "sample_count": 2608, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 162949, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T22:58:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.557, "last_frame_pushed_at": "2026-07-01T22:58:27+09:00", "last_frame_read_age_sec": 4.639, "last_frame_read_at": "2026-07-01T22:58:27+09:00", "mjpeg": {"age_sec": 4.542, "exists": true, "mtime": "2026-07-01T22:58:27.503557+09:00", "mtime_epoch": 1782914307.5035574, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117477}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T22:58:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 943.938, "name": "celery", "pid": 237, "rss_mb": 936.918, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 55, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.834, "pushed": 1.834, "read": 1.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.529, "count": 55, "max_ms": 2.045, "p95_ms": 1.67}, "gstreamer_push": {"avg_ms": 1.691, "count": 55, "max_ms": 6.222, "p95_ms": 5.735}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.569, "count": 55, "max_ms": 78.132, "p95_ms": 73.844}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.155, "count": 55, "max_ms": 83.143, "p95_ms": 80.395}, "yolo11_640_fire_11class": {"avg_ms": 69.367, "count": 55, "max_ms": 78.173, "p95_ms": 77.421}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.698, "count": 55, "max_ms": 86.978, "p95_ms": 86.184}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.166, "count": 55, "max_ms": 97.333, "p95_ms": 96.937}, "read_interval": {"avg_ms": 462.899, "count": 55, "max_ms": 19856.24, "p95_ms": 117.378}, "triton_wait": {"avg_ms": 76.729, "count": 55, "max_ms": 86.99, "p95_ms": 86.196}, "visualize": {"avg_ms": 3.676, "count": 55, "max_ms": 4.466, "p95_ms": 3.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2313}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4105.617, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5590cf791580] error while decoding MB 53 24, bytestream -5 [in#0/rtsp @ 0x5590ceed0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5590ceed0e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:03:43+09:00", "last_error_summary": "[h264 @ 0x5590cf791580] error while decoding MB 53 24, bytestream -5 [in#0/rtsp @ 0x5590ceed0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5590ceed0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:57:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 399.073, "last_started_at": "2026-07-01T22:57:04+09:00", "pid": 28154, "process": {"exists": false, "pid": 28154}, "restart_count": 33, "restart_count_per_hour": 12, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.133, "sample_count": 2474, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277600, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-01T23:03:43+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-01T23:03:43+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-01T23:03:43.428501+09:00", "mtime_epoch": 1782914623.4285007, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91204}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:03:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 999.32, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 34, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.1, "pushed": 1.1, "read": 1.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.489, "count": 33, "max_ms": 4.507, "p95_ms": 4.147}, "gstreamer_push": {"avg_ms": 1.702, "count": 33, "max_ms": 6.588, "p95_ms": 3.856}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.0, "count": 33, "max_ms": 79.929, "p95_ms": 76.504}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.005, "count": 33, "max_ms": 82.901, "p95_ms": 80.466}, "yolo11_640_fire_11class": {"avg_ms": 66.78, "count": 33, "max_ms": 82.146, "p95_ms": 79.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.808, "count": 33, "max_ms": 89.586, "p95_ms": 85.844}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.347, "count": 33, "max_ms": 101.822, "p95_ms": 98.158}, "read_interval": {"avg_ms": 883.368, "count": 34, "max_ms": 16632.385, "p95_ms": 10163.965}, "triton_wait": {"avg_ms": 72.935, "count": 33, "max_ms": 89.598, "p95_ms": 85.857}, "visualize": {"avg_ms": 3.7, "count": 33, "max_ms": 4.305, "p95_ms": 3.871}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2313}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4105.41, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:03:43+09:00", "last_error_summary": "[h264 @ 0x5590cf791580] error while decoding MB 53 24, bytestream -5 [in#0/rtsp @ 0x5590ceed0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5590ceed0e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:03:43+09:00", "last_error_summary": "[h264 @ 0x5590cf791580] error while decoding MB 53 24, bytestream -5 [in#0/rtsp @ 0x5590ceed0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5590ceed0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:03:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 399.091, "last_started_at": "2026-07-01T22:57:04+09:00", "pid": 28154, "process": {"exists": false, "pid": 28154}, "restart_count": 33, "restart_count_per_hour": 12, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.133, "sample_count": 2475, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277601, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T23:03:43+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-01T23:03:43+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-01T23:03:43.428501+09:00", "mtime_epoch": 1782914623.4285007, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91204}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:03:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 999.32, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 33, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.136, "pushed": 1.136, "read": 1.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.487, "count": 34, "max_ms": 4.507, "p95_ms": 4.147}, "gstreamer_push": {"avg_ms": 1.692, "count": 34, "max_ms": 6.588, "p95_ms": 3.856}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.676, "count": 34, "max_ms": 79.929, "p95_ms": 76.504}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.72, "count": 34, "max_ms": 82.901, "p95_ms": 80.466}, "yolo11_640_fire_11class": {"avg_ms": 66.511, "count": 34, "max_ms": 82.146, "p95_ms": 79.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.398, "count": 34, "max_ms": 89.586, "p95_ms": 85.844}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.029, "count": 34, "max_ms": 101.822, "p95_ms": 98.158}, "read_interval": {"avg_ms": 906.943, "count": 33, "max_ms": 16632.385, "p95_ms": 10163.965}, "triton_wait": {"avg_ms": 72.521, "count": 34, "max_ms": 89.598, "p95_ms": 85.857}, "visualize": {"avg_ms": 3.731, "count": 34, "max_ms": 4.769, "p95_ms": 4.305}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2313}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4105.266, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:03:43+09:00", "last_error_summary": "[h264 @ 0x5590cf791580] error while decoding MB 53 24, bytestream -5 [in#0/rtsp @ 0x5590ceed0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5590ceed0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:03:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 399.1, "last_started_at": "2026-07-01T22:57:04+09:00", "pid": 28154, "process": {"exists": false, "pid": 28154}, "restart_count": 33, "restart_count_per_hour": 12, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.133, "sample_count": 2475, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277601, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T23:03:43+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-01T23:03:43+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-01T23:03:43.428501+09:00", "mtime_epoch": 1782914623.4285007, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91204}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:03:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 999.32, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 33, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.136, "pushed": 1.136, "read": 1.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.487, "count": 34, "max_ms": 4.507, "p95_ms": 4.147}, "gstreamer_push": {"avg_ms": 1.692, "count": 34, "max_ms": 6.588, "p95_ms": 3.856}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.676, "count": 34, "max_ms": 79.929, "p95_ms": 76.504}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.72, "count": 34, "max_ms": 82.901, "p95_ms": 80.466}, "yolo11_640_fire_11class": {"avg_ms": 66.511, "count": 34, "max_ms": 82.146, "p95_ms": 79.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.398, "count": 34, "max_ms": 89.586, "p95_ms": 85.844}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.029, "count": 34, "max_ms": 101.822, "p95_ms": 98.158}, "read_interval": {"avg_ms": 906.943, "count": 33, "max_ms": 16632.385, "p95_ms": 10163.965}, "triton_wait": {"avg_ms": 72.521, "count": 34, "max_ms": 89.598, "p95_ms": 85.857}, "visualize": {"avg_ms": 3.731, "count": 34, "max_ms": 4.769, "p95_ms": 4.305}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2313}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4281.355, "limit_mb": 8796093022207.996}, "detail": {"pid": 28413, "restart_count": 34, "start_count": 35}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:03:43+09:00", "last_error_summary": "[h264 @ 0x5590cf791580] error while decoding MB 53 24, bytestream -5 [in#0/rtsp @ 0x5590ceed0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5590ceed0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:03:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:03:46+09:00", "pid": 28413, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.148, "name": "ffmpeg", "pid": 28413, "rss_mb": 204.148, "state": "S (sleeping)", "threads": 106}, "restart_count": 34, "restart_count_per_hour": 13, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.143, "sample_count": 2434, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277601, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:03:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.566, "last_frame_pushed_at": "2026-07-01T23:03:43+09:00", "last_frame_read_age_sec": 4.643, "last_frame_read_at": "2026-07-01T23:03:43+09:00", "mjpeg": {"age_sec": 4.554, "exists": true, "mtime": "2026-07-01T23:03:43.516502+09:00", "mtime_epoch": 1782914623.5165021, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:03:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 999.32, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.826, "pushed": 0.826, "read": 0.826}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.562, "count": 13, "max_ms": 4.507, "p95_ms": 4.507}, "gstreamer_push": {"avg_ms": 1.917, "count": 13, "max_ms": 6.588, "p95_ms": 6.588}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.395, "count": 13, "max_ms": 63.241, "p95_ms": 63.241}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.929, "count": 13, "max_ms": 72.261, "p95_ms": 72.261}, "yolo11_640_fire_11class": {"avg_ms": 56.467, "count": 13, "max_ms": 73.952, "p95_ms": 73.952}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.753, "count": 13, "max_ms": 81.792, "p95_ms": 81.792}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.827, "count": 13, "max_ms": 94.728, "p95_ms": 94.728}, "read_interval": {"avg_ms": 2132.229, "count": 13, "max_ms": 16632.385, "p95_ms": 16632.385}, "triton_wait": {"avg_ms": 58.055, "count": 13, "max_ms": 81.804, "p95_ms": 81.804}, "visualize": {"avg_ms": 3.819, "count": 13, "max_ms": 4.769, "p95_ms": 4.769}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 15.73, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2329}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4121.766, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55d4f6a4c200] missing picture in access unit with size 18 [h264 @ 0x55d4f7579640] no frame! [in#0/rtsp @ 0x55d4f6a47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d4f6a47e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55d4f6a4c840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:07:43+09:00", "last_error_summary": "[NULL @ 0x55d4f6a4c200] missing picture in access unit with size 18 [h264 @ 0x55d4f7579640] no frame! [in#0/rtsp @ 0x55d4f6a47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d4f6a47e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55d4f6a4c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:03:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 237.175, "last_started_at": "2026-07-01T23:03:46+09:00", "pid": 28413, "process": {"exists": false, "pid": 28413}, "restart_count": 34, "restart_count_per_hour": 13, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.118, "sample_count": 2234, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 279528, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:07:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T23:07:43+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T23:07:43+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-01T23:07:43.176394+09:00", "mtime_epoch": 1782914863.1763945, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90691}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:07:43+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1000.035, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 37, "pushed_frames": 37, "read_frames": 37, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.234, "pushed": 1.234, "read": 1.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.38, "count": 37, "max_ms": 3.554, "p95_ms": 3.494}, "gstreamer_push": {"avg_ms": 1.429, "count": 37, "max_ms": 4.109, "p95_ms": 1.557}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.26, "count": 37, "max_ms": 77.651, "p95_ms": 76.905}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.79, "count": 37, "max_ms": 78.773, "p95_ms": 76.914}, "yolo11_640_fire_11class": {"avg_ms": 63.552, "count": 37, "max_ms": 76.607, "p95_ms": 75.843}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.892, "count": 37, "max_ms": 87.0, "p95_ms": 85.7}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.151, "count": 37, "max_ms": 99.138, "p95_ms": 98.19}, "read_interval": {"avg_ms": 811.689, "count": 37, "max_ms": 16280.838, "p95_ms": 10293.159}, "triton_wait": {"avg_ms": 69.96, "count": 37, "max_ms": 87.011, "p95_ms": 85.712}, "visualize": {"avg_ms": 3.682, "count": 37, "max_ms": 3.939, "p95_ms": 3.926}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2329}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4121.77, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:07:43+09:00", "last_error_summary": "[NULL @ 0x55d4f6a4c200] missing picture in access unit with size 18 [h264 @ 0x55d4f7579640] no frame! [in#0/rtsp @ 0x55d4f6a47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d4f6a47e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55d4f6a4c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:03:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 237.19, "last_started_at": "2026-07-01T23:03:46+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 34, "restart_count_per_hour": 13, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.118, "sample_count": 2234, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 279528, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:07:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.021, "last_frame_pushed_at": "2026-07-01T23:07:43+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-01T23:07:43+09:00", "mjpeg": {"age_sec": 0.007, "exists": true, "mtime": "2026-07-01T23:07:43.252396+09:00", "mtime_epoch": 1782914863.2523959, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91607}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:07:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1000.035, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 37, "pushed_frames": 37, "read_frames": 36, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.237, "pushed": 1.237, "read": 1.204}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.38, "count": 37, "max_ms": 3.554, "p95_ms": 3.494}, "gstreamer_push": {"avg_ms": 1.429, "count": 37, "max_ms": 4.109, "p95_ms": 1.557}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.26, "count": 37, "max_ms": 77.651, "p95_ms": 76.905}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.79, "count": 37, "max_ms": 78.773, "p95_ms": 76.914}, "yolo11_640_fire_11class": {"avg_ms": 63.552, "count": 37, "max_ms": 76.607, "p95_ms": 75.843}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.892, "count": 37, "max_ms": 87.0, "p95_ms": 85.7}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.151, "count": 37, "max_ms": 99.138, "p95_ms": 98.19}, "read_interval": {"avg_ms": 831.114, "count": 36, "max_ms": 16280.838, "p95_ms": 10293.159}, "triton_wait": {"avg_ms": 69.96, "count": 37, "max_ms": 87.011, "p95_ms": 85.712}, "visualize": {"avg_ms": 3.682, "count": 37, "max_ms": 3.939, "p95_ms": 3.926}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2329}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4269.715, "limit_mb": 8796093022207.996}, "detail": {"pid": 28565, "restart_count": 35, "start_count": 36}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:07:43+09:00", "last_error_summary": "[NULL @ 0x55d4f6a4c200] missing picture in access unit with size 18 [h264 @ 0x55d4f7579640] no frame! [in#0/rtsp @ 0x55d4f6a47e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d4f6a47e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55d4f6a4c840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:03:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:07:45+09:00", "pid": 28565, "process": {"exists": true, "fd_count": 5, "hwm_mb": 178.422, "name": "ffmpeg", "pid": 28565, "rss_mb": 178.422, "state": "S (sleeping)", "threads": 106}, "restart_count": 35, "restart_count_per_hour": 14, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": true, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.146, "sample_count": 2192, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 279528, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:07:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.658, "last_frame_pushed_at": "2026-07-01T23:07:43+09:00", "last_frame_read_age_sec": 4.721, "last_frame_read_at": "2026-07-01T23:07:43+09:00", "mjpeg": {"age_sec": 4.644, "exists": true, "mtime": "2026-07-01T23:07:43.252396+09:00", "mtime_epoch": 1782914863.2523959, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91607}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:07:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1000.035, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.596, "pushed": 0.596, "read": 0.596}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen", "invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.401, "count": 13, "max_ms": 3.554, "p95_ms": 3.554}, "gstreamer_push": {"avg_ms": 1.591, "count": 13, "max_ms": 4.109, "p95_ms": 4.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.628, "count": 13, "max_ms": 58.9, "p95_ms": 58.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.51, "count": 13, "max_ms": 59.602, "p95_ms": 59.602}, "yolo11_640_fire_11class": {"avg_ms": 47.641, "count": 13, "max_ms": 57.651, "p95_ms": 57.651}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.929, "count": 13, "max_ms": 58.888, "p95_ms": 58.888}}, "truncated_count": 0}, "processing_total": {"avg_ms": 61.317, "count": 13, "max_ms": 71.648, "p95_ms": 71.648}, "read_interval": {"avg_ms": 2107.483, "count": 13, "max_ms": 16280.838, "p95_ms": 16280.838}, "triton_wait": {"avg_ms": 49.101, "count": 13, "max_ms": 59.613, "p95_ms": 59.613}, "visualize": {"avg_ms": 3.711, "count": 13, "max_ms": 3.926, "p95_ms": 3.926}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.826, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4140.75, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563bf187f880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563bf187f880] error while decoding MB 0 33, bytestream 5958 [h264 @ 0x563bf17b0a00] error while decoding MB 32 16, bytestream -24 [in#0/rtsp @ 0x563bf0b8ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563bf0b8ae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 169, "h264_decode_error_count": 231, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:07:52+09:00", "last_error_summary": "[h264 @ 0x563bf187f880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563bf187f880] error while decoding MB 0 33, bytestream 5958 [h264 @ 0x563bf17b0a00] error while decoding MB 32 16, bytestream -24 [in#0/rtsp @ 0x563bf0b8ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563bf0b8ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T22:58:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 562.599, "last_started_at": "2026-07-01T22:58:30+09:00", "pid": 28306, "process": {"exists": false, "pid": 28306}, "restart_count": 193, "restart_count_per_hour": 6, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 194}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.971, "sample_count": 2767, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 168083, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:07:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-01T23:07:52+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-01T23:07:52+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-01T23:07:52.592549+09:00", "mtime_epoch": 1782914872.592549, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136169}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:07:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 946.516, "name": "celery", "pid": 237, "rss_mb": 942.074, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.34, "pushed": 4.34, "read": 4.34}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.568, "count": 130, "max_ms": 2.086, "p95_ms": 1.997}, "gstreamer_push": {"avg_ms": 1.623, "count": 130, "max_ms": 6.213, "p95_ms": 3.776}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.231, "count": 130, "max_ms": 66.294, "p95_ms": 45.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.219, "count": 130, "max_ms": 84.129, "p95_ms": 60.367}, "yolo11_640_fire_11class": {"avg_ms": 47.468, "count": 130, "max_ms": 76.417, "p95_ms": 52.586}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.159, "count": 130, "max_ms": 84.841, "p95_ms": 57.132}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.851, "count": 130, "max_ms": 96.055, "p95_ms": 69.821}, "read_interval": {"avg_ms": 230.449, "count": 130, "max_ms": 10127.504, "p95_ms": 108.871}, "triton_wait": {"avg_ms": 50.612, "count": 130, "max_ms": 84.855, "p95_ms": 60.379}, "visualize": {"avg_ms": 3.659, "count": 130, "max_ms": 4.336, "p95_ms": 3.978}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4137.328, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:07:52+09:00", "last_error_summary": "[h264 @ 0x563bf187f880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563bf187f880] error while decoding MB 0 33, bytestream 5958 [h264 @ 0x563bf17b0a00] error while decoding MB 32 16, bytestream -24 [in#0/rtsp @ 0x563bf0b8ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563bf0b8ae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 169, "h264_decode_error_count": 231, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:07:52+09:00", "last_error_summary": "[h264 @ 0x563bf187f880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563bf187f880] error while decoding MB 0 33, bytestream 5958 [h264 @ 0x563bf17b0a00] error while decoding MB 32 16, bytestream -24 [in#0/rtsp @ 0x563bf0b8ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563bf0b8ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:07:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 562.655, "last_started_at": "2026-07-01T22:58:30+09:00", "pid": 28306, "process": {"exists": false, "pid": 28306}, "restart_count": 193, "restart_count_per_hour": 6, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 194}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.98, "sample_count": 2768, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 168084, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:07:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T23:07:52+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-01T23:07:52+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T23:07:52.592549+09:00", "mtime_epoch": 1782914872.592549, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136169}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:07:52+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 946.516, "name": "celery", "pid": 237, "rss_mb": 936.66, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.335, "pushed": 4.335, "read": 4.335}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.572, "count": 130, "max_ms": 2.086, "p95_ms": 2.008}, "gstreamer_push": {"avg_ms": 1.642, "count": 130, "max_ms": 6.213, "p95_ms": 3.776}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.271, "count": 130, "max_ms": 66.294, "p95_ms": 48.333}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.426, "count": 130, "max_ms": 84.129, "p95_ms": 65.245}, "yolo11_640_fire_11class": {"avg_ms": 47.68, "count": 130, "max_ms": 76.553, "p95_ms": 61.528}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.424, "count": 130, "max_ms": 84.841, "p95_ms": 70.507}}, "truncated_count": 0}, "processing_total": {"avg_ms": 61.135, "count": 130, "max_ms": 97.175, "p95_ms": 81.734}, "read_interval": {"avg_ms": 230.449, "count": 130, "max_ms": 10127.504, "p95_ms": 108.871}, "triton_wait": {"avg_ms": 50.876, "count": 130, "max_ms": 84.855, "p95_ms": 70.521}, "visualize": {"avg_ms": 3.661, "count": 130, "max_ms": 4.336, "p95_ms": 3.978}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4137.48, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 169, "h264_decode_error_count": 231, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:07:52+09:00", "last_error_summary": "[h264 @ 0x563bf187f880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563bf187f880] error while decoding MB 0 33, bytestream 5958 [h264 @ 0x563bf17b0a00] error while decoding MB 32 16, bytestream -24 [in#0/rtsp @ 0x563bf0b8ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563bf0b8ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:07:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 562.663, "last_started_at": "2026-07-01T22:58:30+09:00", "pid": 28306, "process": {"exists": false, "pid": 28306}, "restart_count": 193, "restart_count_per_hour": 6, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 194}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.98, "sample_count": 2768, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 168084, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:07:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T23:07:52+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-01T23:07:52+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-01T23:07:52.592549+09:00", "mtime_epoch": 1782914872.592549, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136169}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:07:52+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 946.516, "name": "celery", "pid": 237, "rss_mb": 936.66, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.333, "pushed": 4.333, "read": 4.333}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.572, "count": 130, "max_ms": 2.086, "p95_ms": 2.008}, "gstreamer_push": {"avg_ms": 1.642, "count": 130, "max_ms": 6.213, "p95_ms": 3.776}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.271, "count": 130, "max_ms": 66.294, "p95_ms": 48.333}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 50.426, "count": 130, "max_ms": 84.129, "p95_ms": 65.245}, "yolo11_640_fire_11class": {"avg_ms": 47.68, "count": 130, "max_ms": 76.553, "p95_ms": 61.528}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.424, "count": 130, "max_ms": 84.841, "p95_ms": 70.507}}, "truncated_count": 0}, "processing_total": {"avg_ms": 61.135, "count": 130, "max_ms": 97.175, "p95_ms": 81.734}, "read_interval": {"avg_ms": 230.449, "count": 130, "max_ms": 10127.504, "p95_ms": 108.871}, "triton_wait": {"avg_ms": 50.876, "count": 130, "max_ms": 84.855, "p95_ms": 70.521}, "visualize": {"avg_ms": 3.661, "count": 130, "max_ms": 4.336, "p95_ms": 3.978}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4257.086, "limit_mb": 8796093022207.996}, "detail": {"pid": 28717, "restart_count": 194, "start_count": 195}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 169, "h264_decode_error_count": 231, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:07:52+09:00", "last_error_summary": "[h264 @ 0x563bf187f880] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563bf187f880] error while decoding MB 0 33, bytestream 5958 [h264 @ 0x563bf17b0a00] error while decoding MB 32 16, bytestream -24 [in#0/rtsp @ 0x563bf0b8ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563bf0b8ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:07:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:07:54+09:00", "pid": 28717, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.238, "name": "ffmpeg", "pid": 28717, "rss_mb": 139.238, "state": "S (sleeping)", "threads": 106}, "restart_count": 194, "restart_count_per_hour": 7, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 195}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.971, "sample_count": 2729, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 168084, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:07:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.272, "last_frame_pushed_at": "2026-07-01T23:07:52+09:00", "last_frame_read_age_sec": 4.373, "last_frame_read_at": "2026-07-01T23:07:52+09:00", "mjpeg": {"age_sec": 4.256, "exists": true, "mtime": "2026-07-01T23:07:52.708551+09:00", "mtime_epoch": 1782914872.7085512, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 137016}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:07:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 946.516, "name": "celery", "pid": 237, "rss_mb": 936.66, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.7, "pushed": 2.7, "read": 2.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.586, "count": 80, "max_ms": 2.037, "p95_ms": 2.012}, "gstreamer_push": {"avg_ms": 1.698, "count": 81, "max_ms": 6.213, "p95_ms": 3.854}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.024, "count": 80, "max_ms": 66.294, "p95_ms": 61.001}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.652, "count": 80, "max_ms": 84.129, "p95_ms": 76.493}, "yolo11_640_fire_11class": {"avg_ms": 48.6, "count": 80, "max_ms": 76.553, "p95_ms": 74.356}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.577, "count": 80, "max_ms": 84.841, "p95_ms": 80.748}}, "truncated_count": 0}, "processing_total": {"avg_ms": 62.513, "count": 81, "max_ms": 97.175, "p95_ms": 92.001}, "read_interval": {"avg_ms": 321.061, "count": 80, "max_ms": 10127.504, "p95_ms": 108.871}, "triton_wait": {"avg_ms": 52.192, "count": 80, "max_ms": 84.855, "p95_ms": 80.761}, "visualize": {"avg_ms": 3.673, "count": 80, "max_ms": 4.336, "p95_ms": 3.928}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1444}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4161.996, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55587ceaf140] error while decoding MB 54 33, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 170, "h264_decode_error_count": 232, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:17:27+09:00", "last_error_summary": "[h264 @ 0x55587ceaf140] error while decoding MB 54 33, bytestream -5", "last_exit_at": "2026-07-01T23:07:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 572.269, "last_started_at": "2026-07-01T23:07:54+09:00", "pid": 28717, "process": {"exists": false, "pid": 28717}, "restart_count": 194, "restart_count_per_hour": 6, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 195}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.982, "sample_count": 2736, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:17:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-01T23:17:27+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-01T23:17:27+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-01T23:17:27.178132+09:00", "mtime_epoch": 1782915447.1781318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 107875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:17:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 949.422, "name": "celery", "pid": 237, "rss_mb": 944.348, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 273, "pushed_frames": 273, "read_frames": 273, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.115, "pushed": 9.115, "read": 9.115}, "hints": ["triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 1.572, "count": 273, "max_ms": 2.191, "p95_ms": 2.009}, "gstreamer_push": {"avg_ms": 1.661, "count": 273, "max_ms": 6.186, "p95_ms": 3.933}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.188, "count": 273, "max_ms": 79.301, "p95_ms": 77.06}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.271, "count": 273, "max_ms": 82.737, "p95_ms": 79.31}, "yolo11_640_fire_11class": {"avg_ms": 74.449, "count": 273, "max_ms": 81.326, "p95_ms": 79.303}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.57, "count": 273, "max_ms": 90.798, "p95_ms": 86.723}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.185, "count": 273, "max_ms": 101.223, "p95_ms": 97.308}, "read_interval": {"avg_ms": 109.867, "count": 273, "max_ms": 125.055, "p95_ms": 114.999}, "triton_wait": {"avg_ms": 82.768, "count": 273, "max_ms": 90.811, "p95_ms": 86.737}, "visualize": {"avg_ms": 3.682, "count": 273, "max_ms": 4.676, "p95_ms": 3.925}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1444}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4161.562, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:17:27+09:00", "last_error_summary": "[h264 @ 0x55587ceaf140] error while decoding MB 54 33, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 170, "h264_decode_error_count": 232, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:17:27+09:00", "last_error_summary": "[h264 @ 0x55587ceaf140] error while decoding MB 54 33, bytestream -5", "last_exit_at": "2026-07-01T23:17:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 572.319, "last_started_at": "2026-07-01T23:07:54+09:00", "pid": 28717, "process": {"exists": false, "pid": 28717}, "restart_count": 194, "restart_count_per_hour": 6, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 195}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.982, "sample_count": 2737, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173293, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:17:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-01T23:17:27+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-01T23:17:27+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-01T23:17:27.178132+09:00", "mtime_epoch": 1782915447.1781318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 107875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:17:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 949.422, "name": "celery", "pid": 237, "rss_mb": 944.348, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 274, "pushed_frames": 274, "read_frames": 273, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.134, "pushed": 9.134, "read": 9.1}, "hints": ["triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 1.571, "count": 273, "max_ms": 2.191, "p95_ms": 2.009}, "gstreamer_push": {"avg_ms": 1.66, "count": 274, "max_ms": 6.186, "p95_ms": 3.933}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.202, "count": 273, "max_ms": 79.301, "p95_ms": 77.06}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.269, "count": 273, "max_ms": 82.737, "p95_ms": 79.31}, "yolo11_640_fire_11class": {"avg_ms": 74.434, "count": 273, "max_ms": 81.326, "p95_ms": 79.303}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.552, "count": 273, "max_ms": 90.798, "p95_ms": 86.614}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.186, "count": 274, "max_ms": 101.223, "p95_ms": 97.308}, "read_interval": {"avg_ms": 109.867, "count": 273, "max_ms": 125.055, "p95_ms": 114.999}, "triton_wait": {"avg_ms": 82.75, "count": 273, "max_ms": 90.811, "p95_ms": 86.628}, "visualize": {"avg_ms": 3.682, "count": 273, "max_ms": 4.676, "p95_ms": 3.925}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1444}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4161.59, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 170, "h264_decode_error_count": 232, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:17:27+09:00", "last_error_summary": "[h264 @ 0x55587ceaf140] error while decoding MB 54 33, bytestream -5", "last_exit_at": "2026-07-01T23:17:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 572.328, "last_started_at": "2026-07-01T23:07:54+09:00", "pid": 28717, "process": {"exists": false, "pid": 28717}, "restart_count": 194, "restart_count_per_hour": 6, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 195}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.982, "sample_count": 2737, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173293, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:17:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-01T23:17:27+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-01T23:17:27+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-01T23:17:27.178132+09:00", "mtime_epoch": 1782915447.1781318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 107875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:17:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 949.422, "name": "celery", "pid": 237, "rss_mb": 944.348, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 273, "pushed_frames": 273, "read_frames": 273, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.102, "pushed": 9.102, "read": 9.102}, "hints": ["triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 1.571, "count": 273, "max_ms": 2.191, "p95_ms": 2.009}, "gstreamer_push": {"avg_ms": 1.661, "count": 273, "max_ms": 6.186, "p95_ms": 3.933}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.202, "count": 273, "max_ms": 79.301, "p95_ms": 77.06}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.269, "count": 273, "max_ms": 82.737, "p95_ms": 79.31}, "yolo11_640_fire_11class": {"avg_ms": 74.434, "count": 273, "max_ms": 81.326, "p95_ms": 79.303}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.552, "count": 273, "max_ms": 90.798, "p95_ms": 86.614}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.167, "count": 273, "max_ms": 101.223, "p95_ms": 97.213}, "read_interval": {"avg_ms": 109.867, "count": 273, "max_ms": 125.055, "p95_ms": 114.999}, "triton_wait": {"avg_ms": 82.75, "count": 273, "max_ms": 90.811, "p95_ms": 86.628}, "visualize": {"avg_ms": 3.682, "count": 273, "max_ms": 4.676, "p95_ms": 3.925}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1444}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4274.645, "limit_mb": 8796093022207.996}, "detail": {"pid": 28824, "restart_count": 195, "start_count": 196}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 170, "h264_decode_error_count": 232, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:17:27+09:00", "last_error_summary": "[h264 @ 0x55587ceaf140] error while decoding MB 54 33, bytestream -5", "last_exit_at": "2026-07-01T23:17:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:17:30+09:00", "pid": 28824, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.602, "name": "ffmpeg", "pid": 28824, "rss_mb": 138.602, "state": "S (sleeping)", "threads": 106}, "restart_count": 195, "restart_count_per_hour": 7, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 196}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.976, "sample_count": 2692, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173293, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:17:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 5.036, "last_frame_pushed_at": "2026-07-01T23:17:27+09:00", "last_frame_read_age_sec": 5.132, "last_frame_read_at": "2026-07-01T23:17:27+09:00", "mjpeg": {"age_sec": 5.021, "exists": true, "mtime": "2026-07-01T23:17:27.290134+09:00", "mtime_epoch": 1782915447.2901337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 108012}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:17:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 949.422, "name": "celery", "pid": 237, "rss_mb": 944.348, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 227, "pushed_frames": 227, "read_frames": 226, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.582, "pushed": 7.582, "read": 7.549}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 1.557, "count": 227, "max_ms": 2.191, "p95_ms": 1.77}, "gstreamer_push": {"avg_ms": 1.667, "count": 227, "max_ms": 6.186, "p95_ms": 3.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.168, "count": 227, "max_ms": 79.301, "p95_ms": 77.135}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.577, "count": 227, "max_ms": 82.737, "p95_ms": 79.456}, "yolo11_640_fire_11class": {"avg_ms": 74.681, "count": 227, "max_ms": 81.326, "p95_ms": 79.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.922, "count": 227, "max_ms": 90.798, "p95_ms": 86.723}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.482, "count": 227, "max_ms": 101.223, "p95_ms": 97.335}, "read_interval": {"avg_ms": 110.186, "count": 226, "max_ms": 125.055, "p95_ms": 115.424}, "triton_wait": {"avg_ms": 83.09, "count": 227, "max_ms": 90.811, "p95_ms": 86.737}, "visualize": {"avg_ms": 3.676, "count": 227, "max_ms": 4.676, "p95_ms": 3.946}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2392}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4132.938, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e67b0f3180] error while decoding MB 91 23, bytestream -7 [in#0/rtsp @ 0x55e67b06be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e67b06be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:22:00+09:00", "last_error_summary": "[h264 @ 0x55e67b0f3180] error while decoding MB 91 23, bytestream -7 [in#0/rtsp @ 0x55e67b06be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e67b06be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:03:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 855.008, "last_started_at": "2026-07-01T23:07:45+09:00", "pid": 28565, "process": {"exists": false, "pid": 28565}, "restart_count": 35, "restart_count_per_hour": 11, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.888, "sample_count": 2523, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 287117, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:22:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-01T23:22:00+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T23:22:00+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-01T23:22:00.842755+09:00", "mtime_epoch": 1782915720.8427553, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92879}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:22:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1003.645, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 39, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.3, "pushed": 1.3, "read": 1.3}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.439, "count": 39, "max_ms": 4.251, "p95_ms": 3.557}, "gstreamer_push": {"avg_ms": 1.329, "count": 39, "max_ms": 1.397, "p95_ms": 1.389}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.26, "count": 39, "max_ms": 73.367, "p95_ms": 71.723}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.837, "count": 39, "max_ms": 78.232, "p95_ms": 77.506}, "yolo11_640_fire_11class": {"avg_ms": 67.953, "count": 39, "max_ms": 82.695, "p95_ms": 78.684}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.618, "count": 39, "max_ms": 85.001, "p95_ms": 84.925}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.166, "count": 39, "max_ms": 97.3, "p95_ms": 97.112}, "read_interval": {"avg_ms": 769.904, "count": 39, "max_ms": 15966.529, "p95_ms": 10262.597}, "triton_wait": {"avg_ms": 73.288, "count": 39, "max_ms": 85.014, "p95_ms": 84.937}, "visualize": {"avg_ms": 3.645, "count": 39, "max_ms": 3.878, "p95_ms": 3.82}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2392}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4132.855, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:22:00+09:00", "last_error_summary": "[h264 @ 0x55e67b0f3180] error while decoding MB 91 23, bytestream -7 [in#0/rtsp @ 0x55e67b06be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e67b06be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:22:00+09:00", "last_error_summary": "[h264 @ 0x55e67b0f3180] error while decoding MB 91 23, bytestream -7 [in#0/rtsp @ 0x55e67b06be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e67b06be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:22:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 855.027, "last_started_at": "2026-07-01T23:07:45+09:00", "pid": 28565, "process": {"exists": false, "pid": 28565}, "restart_count": 35, "restart_count_per_hour": 11, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.888, "sample_count": 2524, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 287117, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:22:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-01T23:22:00+09:00", "last_frame_read_age_sec": 0.083, "last_frame_read_at": "2026-07-01T23:22:00+09:00", "mjpeg": {"age_sec": 0.079, "exists": true, "mtime": "2026-07-01T23:22:00.842755+09:00", "mtime_epoch": 1782915720.8427553, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92879}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:22:00+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1003.645, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 39, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.3, "pushed": 1.3, "read": 1.3}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.439, "count": 39, "max_ms": 4.251, "p95_ms": 3.557}, "gstreamer_push": {"avg_ms": 1.331, "count": 39, "max_ms": 1.435, "p95_ms": 1.397}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.26, "count": 39, "max_ms": 73.367, "p95_ms": 71.723}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.837, "count": 39, "max_ms": 78.232, "p95_ms": 77.506}, "yolo11_640_fire_11class": {"avg_ms": 67.953, "count": 39, "max_ms": 82.695, "p95_ms": 78.684}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.618, "count": 39, "max_ms": 85.001, "p95_ms": 84.925}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.67, "count": 39, "max_ms": 97.3, "p95_ms": 97.112}, "read_interval": {"avg_ms": 769.904, "count": 39, "max_ms": 15966.529, "p95_ms": 10262.597}, "triton_wait": {"avg_ms": 73.288, "count": 39, "max_ms": 85.014, "p95_ms": 84.937}, "visualize": {"avg_ms": 3.645, "count": 39, "max_ms": 3.878, "p95_ms": 3.82}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2392}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4132.953, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:22:00+09:00", "last_error_summary": "[h264 @ 0x55e67b0f3180] error while decoding MB 91 23, bytestream -7 [in#0/rtsp @ 0x55e67b06be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e67b06be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:22:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 855.035, "last_started_at": "2026-07-01T23:07:45+09:00", "pid": 28565, "process": {"exists": false, "pid": 28565}, "restart_count": 35, "restart_count_per_hour": 11, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.888, "sample_count": 2524, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 287117, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:22:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-01T23:22:00+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-01T23:22:00+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-01T23:22:00.842755+09:00", "mtime_epoch": 1782915720.8427553, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92879}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:22:00+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1003.645, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 38, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.304, "pushed": 1.304, "read": 1.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.439, "count": 39, "max_ms": 4.251, "p95_ms": 3.557}, "gstreamer_push": {"avg_ms": 1.331, "count": 39, "max_ms": 1.435, "p95_ms": 1.397}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.26, "count": 39, "max_ms": 73.367, "p95_ms": 71.723}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.837, "count": 39, "max_ms": 78.232, "p95_ms": 77.506}, "yolo11_640_fire_11class": {"avg_ms": 67.953, "count": 39, "max_ms": 82.695, "p95_ms": 78.684}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.618, "count": 39, "max_ms": 85.001, "p95_ms": 84.925}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.67, "count": 39, "max_ms": 97.3, "p95_ms": 97.112}, "read_interval": {"avg_ms": 787.262, "count": 38, "max_ms": 15966.529, "p95_ms": 10262.597}, "triton_wait": {"avg_ms": 73.288, "count": 39, "max_ms": 85.014, "p95_ms": 84.937}, "visualize": {"avg_ms": 3.645, "count": 39, "max_ms": 3.878, "p95_ms": 3.82}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2392}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4215.133, "limit_mb": 8796093022207.996}, "detail": {"pid": 28931, "restart_count": 36, "start_count": 37}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:22:00+09:00", "last_error_summary": "[h264 @ 0x55e67b0f3180] error while decoding MB 91 23, bytestream -7 [in#0/rtsp @ 0x55e67b06be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e67b06be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:22:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:22:03+09:00", "pid": 28931, "process": {"exists": true, "fd_count": 5, "hwm_mb": 115.531, "name": "ffmpeg", "pid": 28931, "rss_mb": 115.531, "state": "S (sleeping)", "threads": 19}, "restart_count": 36, "restart_count_per_hour": 12, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.891, "sample_count": 2480, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 287117, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:22:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.771, "last_frame_pushed_at": "2026-07-01T23:22:00+09:00", "last_frame_read_age_sec": 4.847, "last_frame_read_at": "2026-07-01T23:22:00+09:00", "mjpeg": {"age_sec": 4.755, "exists": true, "mtime": "2026-07-01T23:22:00.930757+09:00", "mtime_epoch": 1782915720.9307568, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93131}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:22:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1003.645, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.554, "pushed": 0.554, "read": 0.554}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.48, "count": 12, "max_ms": 4.251, "p95_ms": 4.251}, "gstreamer_push": {"avg_ms": 1.328, "count": 12, "max_ms": 1.435, "p95_ms": 1.435}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.985, "count": 12, "max_ms": 56.347, "p95_ms": 56.347}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.986, "count": 12, "max_ms": 57.628, "p95_ms": 57.628}, "yolo11_640_fire_11class": {"avg_ms": 55.763, "count": 12, "max_ms": 59.132, "p95_ms": 59.132}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.828, "count": 12, "max_ms": 57.28, "p95_ms": 57.28}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.582, "count": 12, "max_ms": 74.274, "p95_ms": 74.274}, "read_interval": {"avg_ms": 2255.264, "count": 12, "max_ms": 15966.529, "p95_ms": 15966.529}, "triton_wait": {"avg_ms": 55.915, "count": 12, "max_ms": 59.145, "p95_ms": 59.145}, "visualize": {"avg_ms": 3.651, "count": 12, "max_ms": 3.878, "p95_ms": 3.878}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.648, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1468}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4162.211, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5611d1b39100] left block unavailable for requested intra mode [h264 @ 0x5611d1b39100] error while decoding MB 0 32, bytestream 13391 [h264 @ 0x5611d1cd6e00] error while decoding MB 55 38, bytestream -5 [in#0/rtsp @ 0x5611d0ff5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611d0ff5e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 171, "h264_decode_error_count": 234, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:22:53+09:00", "last_error_summary": "[h264 @ 0x5611d1b39100] left block unavailable for requested intra mode [h264 @ 0x5611d1b39100] error while decoding MB 0 32, bytestream 13391 [h264 @ 0x5611d1cd6e00] error while decoding MB 55 38, bytestream -5 [in#0/rtsp @ 0x5611d0ff5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611d0ff5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:17:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 322.921, "last_started_at": "2026-07-01T23:17:30+09:00", "pid": 28824, "process": {"exists": false, "pid": 28824}, "restart_count": 195, "restart_count_per_hour": 6, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 196}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.926, "sample_count": 2686, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 176177, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:22:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-01T23:22:53+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-01T23:22:53+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-01T23:22:53.175642+09:00", "mtime_epoch": 1782915773.1756418, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 108828}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:22:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 949.422, "name": "celery", "pid": 237, "rss_mb": 945.754, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.372, "pushed": 3.372, "read": 3.372}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.55, "count": 101, "max_ms": 2.112, "p95_ms": 1.605}, "gstreamer_push": {"avg_ms": 1.64, "count": 101, "max_ms": 4.065, "p95_ms": 3.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.772, "count": 101, "max_ms": 78.474, "p95_ms": 76.825}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.028, "count": 101, "max_ms": 83.49, "p95_ms": 78.42}, "yolo11_640_fire_11class": {"avg_ms": 73.315, "count": 101, "max_ms": 80.348, "p95_ms": 78.485}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.111, "count": 101, "max_ms": 88.435, "p95_ms": 86.309}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.568, "count": 101, "max_ms": 98.722, "p95_ms": 97.287}, "read_interval": {"avg_ms": 296.932, "count": 101, "max_ms": 19151.391, "p95_ms": 116.6}, "triton_wait": {"avg_ms": 81.149, "count": 101, "max_ms": 88.447, "p95_ms": 86.323}, "visualize": {"avg_ms": 3.716, "count": 101, "max_ms": 4.907, "p95_ms": 3.985}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1468}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4164.508, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:22:53+09:00", "last_error_summary": "[h264 @ 0x5611d1b39100] left block unavailable for requested intra mode [h264 @ 0x5611d1b39100] error while decoding MB 0 32, bytestream 13391 [h264 @ 0x5611d1cd6e00] error while decoding MB 55 38, bytestream -5 [in#0/rtsp @ 0x5611d0ff5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611d0ff5e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 171, "h264_decode_error_count": 234, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:22:53+09:00", "last_error_summary": "[h264 @ 0x5611d1b39100] left block unavailable for requested intra mode [h264 @ 0x5611d1b39100] error while decoding MB 0 32, bytestream 13391 [h264 @ 0x5611d1cd6e00] error while decoding MB 55 38, bytestream -5 [in#0/rtsp @ 0x5611d0ff5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611d0ff5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:22:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 322.961, "last_started_at": "2026-07-01T23:17:30+09:00", "pid": 28824, "process": {"exists": false, "pid": 28824}, "restart_count": 195, "restart_count_per_hour": 6, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 196}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.926, "sample_count": 2687, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 176178, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:22:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T23:22:53+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-01T23:22:53+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-01T23:22:53.175642+09:00", "mtime_epoch": 1782915773.1756418, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 108828}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:22:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 949.422, "name": "celery", "pid": 237, "rss_mb": 945.754, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.401, "pushed": 3.401, "read": 3.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.549, "count": 102, "max_ms": 2.112, "p95_ms": 1.605}, "gstreamer_push": {"avg_ms": 1.639, "count": 102, "max_ms": 4.065, "p95_ms": 3.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.595, "count": 102, "max_ms": 78.474, "p95_ms": 76.825}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.087, "count": 102, "max_ms": 83.49, "p95_ms": 78.495}, "yolo11_640_fire_11class": {"avg_ms": 73.317, "count": 102, "max_ms": 80.348, "p95_ms": 78.485}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.933, "count": 102, "max_ms": 88.435, "p95_ms": 86.309}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.555, "count": 102, "max_ms": 98.722, "p95_ms": 97.287}, "read_interval": {"avg_ms": 296.932, "count": 101, "max_ms": 19151.391, "p95_ms": 116.6}, "triton_wait": {"avg_ms": 81.138, "count": 102, "max_ms": 88.447, "p95_ms": 86.323}, "visualize": {"avg_ms": 3.716, "count": 102, "max_ms": 4.907, "p95_ms": 3.985}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1468}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4164.543, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 171, "h264_decode_error_count": 234, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:22:53+09:00", "last_error_summary": "[h264 @ 0x5611d1b39100] left block unavailable for requested intra mode [h264 @ 0x5611d1b39100] error while decoding MB 0 32, bytestream 13391 [h264 @ 0x5611d1cd6e00] error while decoding MB 55 38, bytestream -5 [in#0/rtsp @ 0x5611d0ff5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611d0ff5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:22:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 322.969, "last_started_at": "2026-07-01T23:17:30+09:00", "pid": 28824, "process": {"exists": false, "pid": 28824}, "restart_count": 195, "restart_count_per_hour": 6, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 196}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.926, "sample_count": 2687, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 176178, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:22:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T23:22:53+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-01T23:22:53+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-01T23:22:53.175642+09:00", "mtime_epoch": 1782915773.1756418, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 108828}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:22:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 949.422, "name": "celery", "pid": 237, "rss_mb": 946.012, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.368, "pushed": 3.368, "read": 3.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.549, "count": 101, "max_ms": 2.112, "p95_ms": 1.605}, "gstreamer_push": {"avg_ms": 1.641, "count": 101, "max_ms": 4.065, "p95_ms": 3.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.568, "count": 101, "max_ms": 78.474, "p95_ms": 76.825}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.084, "count": 101, "max_ms": 83.49, "p95_ms": 78.495}, "yolo11_640_fire_11class": {"avg_ms": 73.318, "count": 101, "max_ms": 80.348, "p95_ms": 78.485}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.908, "count": 101, "max_ms": 88.435, "p95_ms": 86.309}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.533, "count": 101, "max_ms": 98.722, "p95_ms": 97.287}, "read_interval": {"avg_ms": 296.932, "count": 101, "max_ms": 19151.391, "p95_ms": 116.6}, "triton_wait": {"avg_ms": 81.115, "count": 101, "max_ms": 88.447, "p95_ms": 86.323}, "visualize": {"avg_ms": 3.716, "count": 101, "max_ms": 4.907, "p95_ms": 3.985}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1468}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4279.297, "limit_mb": 8796093022207.996}, "detail": {"pid": 29083, "restart_count": 196, "start_count": 197}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 171, "h264_decode_error_count": 234, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:22:53+09:00", "last_error_summary": "[h264 @ 0x5611d1b39100] left block unavailable for requested intra mode [h264 @ 0x5611d1b39100] error while decoding MB 0 32, bytestream 13391 [h264 @ 0x5611d1cd6e00] error while decoding MB 55 38, bytestream -5 [in#0/rtsp @ 0x5611d0ff5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611d0ff5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:22:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:22:55+09:00", "pid": 29083, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.078, "name": "ffmpeg", "pid": 29083, "rss_mb": 144.078, "state": "S (sleeping)", "threads": 106}, "restart_count": 196, "restart_count_per_hour": 7, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 197}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.886, "sample_count": 2649, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 176178, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:22:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.231, "last_frame_pushed_at": "2026-07-01T23:22:53+09:00", "last_frame_read_age_sec": 4.324, "last_frame_read_at": "2026-07-01T23:22:53+09:00", "mjpeg": {"age_sec": 4.216, "exists": true, "mtime": "2026-07-01T23:22:53.279644+09:00", "mtime_epoch": 1782915773.2796435, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 109767}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:22:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 949.422, "name": "celery", "pid": 237, "rss_mb": 946.012, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 63, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.1, "pushed": 2.1, "read": 2.1}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.55, "count": 63, "max_ms": 2.112, "p95_ms": 1.783}, "gstreamer_push": {"avg_ms": 1.623, "count": 63, "max_ms": 4.065, "p95_ms": 3.865}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.802, "count": 63, "max_ms": 78.474, "p95_ms": 76.792}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.15, "count": 63, "max_ms": 83.49, "p95_ms": 79.726}, "yolo11_640_fire_11class": {"avg_ms": 72.707, "count": 63, "max_ms": 80.348, "p95_ms": 78.568}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.526, "count": 63, "max_ms": 88.435, "p95_ms": 86.303}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.221, "count": 63, "max_ms": 98.634, "p95_ms": 97.287}, "read_interval": {"avg_ms": 409.244, "count": 63, "max_ms": 19151.391, "p95_ms": 116.6}, "triton_wait": {"avg_ms": 79.851, "count": 63, "max_ms": 88.447, "p95_ms": 86.314}, "visualize": {"avg_ms": 3.703, "count": 63, "max_ms": 4.907, "p95_ms": 3.977}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2405}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4114.184, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x563141573200] missing picture in access unit with size 18 [h264 @ 0x563141d5a600] no frame! [in#0/rtsp @ 0x56314156ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56314156ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563141573840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:25:20+09:00", "last_error_summary": "[NULL @ 0x563141573200] missing picture in access unit with size 18 [h264 @ 0x563141d5a600] no frame! [in#0/rtsp @ 0x56314156ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56314156ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563141573840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:22:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.487, "last_started_at": "2026-07-01T23:22:03+09:00", "pid": 28931, "process": {"exists": false, "pid": 28931}, "restart_count": 36, "restart_count_per_hour": 11, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.014, "sample_count": 2194, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288624, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:25:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.083, "last_frame_pushed_at": "2026-07-01T23:25:20+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-01T23:25:20+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-01T23:25:20.106134+09:00", "mtime_epoch": 1782915920.1061337, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93969}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:25:20+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.477, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.437, "pushed": 0.437, "read": 0.473}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.41, "count": 12, "max_ms": 3.512, "p95_ms": 3.512}, "gstreamer_push": {"avg_ms": 1.773, "count": 12, "max_ms": 4.367, "p95_ms": 4.367}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.121, "count": 12, "max_ms": 66.374, "p95_ms": 66.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.904, "count": 12, "max_ms": 83.564, "p95_ms": 83.564}, "yolo11_640_fire_11class": {"avg_ms": 65.497, "count": 12, "max_ms": 77.326, "p95_ms": 77.326}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.5, "count": 12, "max_ms": 87.285, "p95_ms": 87.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.16, "count": 12, "max_ms": 99.75, "p95_ms": 99.75}, "read_interval": {"avg_ms": 2857.295, "count": 13, "max_ms": 26294.329, "p95_ms": 26294.329}, "triton_wait": {"avg_ms": 71.891, "count": 12, "max_ms": 87.298, "p95_ms": 87.298}, "visualize": {"avg_ms": 3.624, "count": 12, "max_ms": 3.713, "p95_ms": 3.713}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.474, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2405}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4113.566, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:25:20+09:00", "last_error_summary": "[NULL @ 0x563141573200] missing picture in access unit with size 18 [h264 @ 0x563141d5a600] no frame! [in#0/rtsp @ 0x56314156ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56314156ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563141573840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:25:20+09:00", "last_error_summary": "[NULL @ 0x563141573200] missing picture in access unit with size 18 [h264 @ 0x563141d5a600] no frame! [in#0/rtsp @ 0x56314156ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56314156ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563141573840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:25:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.523, "last_started_at": "2026-07-01T23:22:03+09:00", "pid": 28931, "process": {"exists": false, "pid": 28931}, "restart_count": 36, "restart_count_per_hour": 11, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.014, "sample_count": 2195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288625, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:25:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T23:25:20+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-01T23:25:20+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-01T23:25:20.106134+09:00", "mtime_epoch": 1782915920.1061337, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93969}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:25:20+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.477, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.473, "pushed": 0.473, "read": 0.473}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.402, "count": 13, "max_ms": 3.512, "p95_ms": 3.512}, "gstreamer_push": {"avg_ms": 1.738, "count": 13, "max_ms": 4.367, "p95_ms": 4.367}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.834, "count": 13, "max_ms": 75.383, "p95_ms": 75.383}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.401, "count": 13, "max_ms": 83.564, "p95_ms": 83.564}, "yolo11_640_fire_11class": {"avg_ms": 66.16, "count": 13, "max_ms": 77.326, "p95_ms": 77.326}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.363, "count": 13, "max_ms": 87.285, "p95_ms": 87.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.99, "count": 13, "max_ms": 99.75, "p95_ms": 99.75}, "read_interval": {"avg_ms": 2857.295, "count": 13, "max_ms": 26294.329, "p95_ms": 26294.329}, "triton_wait": {"avg_ms": 72.725, "count": 13, "max_ms": 87.298, "p95_ms": 87.298}, "visualize": {"avg_ms": 3.627, "count": 13, "max_ms": 3.713, "p95_ms": 3.713}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.51, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2405}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4113.668, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:25:20+09:00", "last_error_summary": "[NULL @ 0x563141573200] missing picture in access unit with size 18 [h264 @ 0x563141d5a600] no frame! [in#0/rtsp @ 0x56314156ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56314156ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563141573840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:25:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 196.53, "last_started_at": "2026-07-01T23:22:03+09:00", "pid": 28931, "process": {"exists": false, "pid": 28931}, "restart_count": 36, "restart_count_per_hour": 11, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.014, "sample_count": 2195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288625, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:25:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T23:25:20+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T23:25:20+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T23:25:20.106134+09:00", "mtime_epoch": 1782915920.1061337, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93969}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:25:20+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.477, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.472, "pushed": 0.472, "read": 0.472}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.402, "count": 13, "max_ms": 3.512, "p95_ms": 3.512}, "gstreamer_push": {"avg_ms": 1.738, "count": 13, "max_ms": 4.367, "p95_ms": 4.367}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.834, "count": 13, "max_ms": 75.383, "p95_ms": 75.383}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.401, "count": 13, "max_ms": 83.564, "p95_ms": 83.564}, "yolo11_640_fire_11class": {"avg_ms": 66.16, "count": 13, "max_ms": 77.326, "p95_ms": 77.326}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.363, "count": 13, "max_ms": 87.285, "p95_ms": 87.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.99, "count": 13, "max_ms": 99.75, "p95_ms": 99.75}, "read_interval": {"avg_ms": 2857.295, "count": 13, "max_ms": 26294.329, "p95_ms": 26294.329}, "triton_wait": {"avg_ms": 72.725, "count": 13, "max_ms": 87.298, "p95_ms": 87.298}, "visualize": {"avg_ms": 3.627, "count": 13, "max_ms": 3.713, "p95_ms": 3.713}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.517, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2405}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4136.641, "limit_mb": 8796093022207.996}, "detail": {"pid": 29190, "restart_count": 37, "start_count": 38}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:25:20+09:00", "last_error_summary": "[NULL @ 0x563141573200] missing picture in access unit with size 18 [h264 @ 0x563141d5a600] no frame! [in#0/rtsp @ 0x56314156ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56314156ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563141573840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:25:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:25:22+09:00", "pid": 29190, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.023, "name": "ffmpeg", "pid": 29190, "rss_mb": 58.023, "state": "S (sleeping)", "threads": 1}, "restart_count": 37, "restart_count_per_hour": 12, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.0, "sample_count": 2156, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288625, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:25:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.317, "last_frame_pushed_at": "2026-07-01T23:25:20+09:00", "last_frame_read_age_sec": 4.414, "last_frame_read_at": "2026-07-01T23:25:20+09:00", "mjpeg": {"age_sec": 4.302, "exists": true, "mtime": "2026-07-01T23:25:20.218136+09:00", "mtime_epoch": 1782915920.2181356, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95879}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:25:24+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.477, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.171, "pushed": 2.171, "read": 2.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.405, "count": 12, "max_ms": 3.512, "p95_ms": 3.512}, "gstreamer_push": {"avg_ms": 1.519, "count": 12, "max_ms": 3.539, "p95_ms": 3.539}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.183, "count": 12, "max_ms": 75.383, "p95_ms": 75.383}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.158, "count": 12, "max_ms": 83.564, "p95_ms": 83.564}, "yolo11_640_fire_11class": {"avg_ms": 67.743, "count": 12, "max_ms": 77.326, "p95_ms": 77.326}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.323, "count": 12, "max_ms": 87.285, "p95_ms": 87.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.993, "count": 12, "max_ms": 99.75, "p95_ms": 99.75}, "read_interval": {"avg_ms": 2283.9, "count": 12, "max_ms": 26294.329, "p95_ms": 26294.329}, "triton_wait": {"avg_ms": 74.715, "count": 12, "max_ms": 87.298, "p95_ms": 87.298}, "visualize": {"avg_ms": 3.624, "count": 12, "max_ms": 3.713, "p95_ms": 3.713}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.527, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2414}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4421.438, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:25:20+09:00", "last_error_summary": "[NULL @ 0x563141573200] missing picture in access unit with size 18 [h264 @ 0x563141d5a600] no frame! [in#0/rtsp @ 0x56314156ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56314156ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563141573840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:25:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.18, "last_started_at": "2026-07-01T23:25:22+09:00", "pid": 29190, "process": {"exists": true, "fd_count": 5, "hwm_mb": 328.691, "name": "ffmpeg", "pid": 29190, "rss_mb": 328.691, "state": "S (sleeping)", "threads": 106}, "restart_count": 37, "restart_count_per_hour": 12, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.584, "sample_count": 2107, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 289697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:27:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.059, "last_frame_pushed_at": "2026-07-01T23:27:22+09:00", "last_frame_read_age_sec": 30.158, "last_frame_read_at": "2026-07-01T23:27:22+09:00", "mjpeg": {"age_sec": 30.046, "exists": true, "mtime": "2026-07-01T23:27:22.652364+09:00", "mtime_epoch": 1782916042.6523638, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86868}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:27:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.992, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2414}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4409.543, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:25:20+09:00", "last_error_summary": "[NULL @ 0x563141573200] missing picture in access unit with size 18 [h264 @ 0x563141d5a600] no frame! [in#0/rtsp @ 0x56314156ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56314156ee00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563141573840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:25:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.193, "last_started_at": "2026-07-01T23:25:22+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 37, "restart_count_per_hour": 12, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.584, "sample_count": 2107, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 289697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:27:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.073, "last_frame_pushed_at": "2026-07-01T23:27:22+09:00", "last_frame_read_age_sec": 30.171, "last_frame_read_at": "2026-07-01T23:27:22+09:00", "mjpeg": {"age_sec": 30.06, "exists": true, "mtime": "2026-07-01T23:27:22.652364+09:00", "mtime_epoch": 1782916042.6523638, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86868}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:27:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.992, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2414}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4120.73, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d57d669fc0] error while decoding MB 71 18, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:27:52+09:00", "last_error_summary": "[h264 @ 0x55d57d669fc0] error while decoding MB 71 18, bytestream -5", "last_exit_at": "2026-07-01T23:25:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 150.251, "last_started_at": "2026-07-01T23:25:22+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 37, "restart_count_per_hour": 12, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.584, "sample_count": 2107, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 289697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:27:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.13, "last_frame_pushed_at": "2026-07-01T23:27:22+09:00", "last_frame_read_age_sec": 30.229, "last_frame_read_at": "2026-07-01T23:27:22+09:00", "mjpeg": {"age_sec": 30.117, "exists": true, "mtime": "2026-07-01T23:27:22.652364+09:00", "mtime_epoch": 1782916042.6523638, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86868}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:27:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.992, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2414}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4289.172, "limit_mb": 8796093022207.996}, "detail": {"pid": 29342, "restart_count": 38, "start_count": 39}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 39, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:27:52+09:00", "last_error_summary": "[h264 @ 0x55d57d669fc0] error while decoding MB 71 18, bytestream -5", "last_exit_at": "2026-07-01T23:25:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:27:54+09:00", "pid": 29342, "process": {"exists": true, "fd_count": 5, "hwm_mb": 198.031, "name": "ffmpeg", "pid": 29342, "rss_mb": 198.031, "state": "S (sleeping)", "threads": 106}, "restart_count": 38, "restart_count_per_hour": 13, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.616, "sample_count": 2063, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 289697, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:27:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 34.349, "last_frame_pushed_at": "2026-07-01T23:27:22+09:00", "last_frame_read_age_sec": 34.448, "last_frame_read_at": "2026-07-01T23:27:22+09:00", "mjpeg": {"age_sec": 34.336, "exists": true, "mtime": "2026-07-01T23:27:22.652364+09:00", "mtime_epoch": 1782916042.6523638, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86868}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:27:56+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.992, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2429}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4131.461, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b3fbc1fd80] error while decoding MB 101 17, bytestream -5 [in#0/rtsp @ 0x55b3fb28de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b3fb28de00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:31:39+09:00", "last_error_summary": "[h264 @ 0x55b3fbc1fd80] error while decoding MB 101 17, bytestream -5 [in#0/rtsp @ 0x55b3fb28de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b3fb28de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:25:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 224.766, "last_started_at": "2026-07-01T23:27:54+09:00", "pid": 29342, "process": {"exists": false, "pid": 29342}, "restart_count": 38, "restart_count_per_hour": 11, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.688, "sample_count": 2176, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 291483, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:31:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.085, "last_frame_pushed_at": "2026-07-01T23:31:39+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-01T23:31:39+09:00", "mjpeg": {"age_sec": 0.072, "exists": true, "mtime": "2026-07-01T23:31:39.681383+09:00", "mtime_epoch": 1782916299.6813831, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91799}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:31:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1006.023, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 39, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.303, "pushed": 1.303, "read": 1.303}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.867, "count": 39, "max_ms": 5.701, "p95_ms": 5.686}, "gstreamer_push": {"avg_ms": 1.806, "count": 39, "max_ms": 7.072, "p95_ms": 4.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.709, "count": 39, "max_ms": 78.743, "p95_ms": 76.344}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.316, "count": 39, "max_ms": 79.374, "p95_ms": 79.245}, "yolo11_640_fire_11class": {"avg_ms": 71.914, "count": 39, "max_ms": 79.879, "p95_ms": 78.074}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.489, "count": 39, "max_ms": 87.343, "p95_ms": 85.47}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.169, "count": 39, "max_ms": 101.546, "p95_ms": 101.198}, "read_interval": {"avg_ms": 768.052, "count": 39, "max_ms": 16024.735, "p95_ms": 9804.317}, "triton_wait": {"avg_ms": 80.502, "count": 39, "max_ms": 87.354, "p95_ms": 85.482}, "visualize": {"avg_ms": 4.171, "count": 39, "max_ms": 6.45, "p95_ms": 6.236}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2429}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4131.039, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:31:39+09:00", "last_error_summary": "[h264 @ 0x55b3fbc1fd80] error while decoding MB 101 17, bytestream -5 [in#0/rtsp @ 0x55b3fb28de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b3fb28de00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:31:39+09:00", "last_error_summary": "[h264 @ 0x55b3fbc1fd80] error while decoding MB 101 17, bytestream -5 [in#0/rtsp @ 0x55b3fb28de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b3fb28de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:31:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 224.798, "last_started_at": "2026-07-01T23:27:54+09:00", "pid": 29342, "process": {"exists": false, "pid": 29342}, "restart_count": 38, "restart_count_per_hour": 11, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.688, "sample_count": 2176, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 291484, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:31:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T23:31:39+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-01T23:31:39+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-01T23:31:39.681383+09:00", "mtime_epoch": 1782916299.6813831, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91799}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:31:39+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1006.023, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.335, "pushed": 1.335, "read": 1.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.852, "count": 40, "max_ms": 5.701, "p95_ms": 5.653}, "gstreamer_push": {"avg_ms": 1.799, "count": 40, "max_ms": 7.072, "p95_ms": 4.315}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.966, "count": 40, "max_ms": 78.743, "p95_ms": 74.004}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.387, "count": 40, "max_ms": 79.374, "p95_ms": 78.33}, "yolo11_640_fire_11class": {"avg_ms": 71.95, "count": 40, "max_ms": 79.879, "p95_ms": 77.728}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.53, "count": 40, "max_ms": 87.343, "p95_ms": 85.082}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.173, "count": 40, "max_ms": 101.546, "p95_ms": 101.067}, "read_interval": {"avg_ms": 768.052, "count": 39, "max_ms": 16024.735, "p95_ms": 9804.317}, "triton_wait": {"avg_ms": 80.542, "count": 40, "max_ms": 87.354, "p95_ms": 85.095}, "visualize": {"avg_ms": 4.158, "count": 40, "max_ms": 6.45, "p95_ms": 6.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2429}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4130.914, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:31:39+09:00", "last_error_summary": "[h264 @ 0x55b3fbc1fd80] error while decoding MB 101 17, bytestream -5 [in#0/rtsp @ 0x55b3fb28de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b3fb28de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:31:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 224.805, "last_started_at": "2026-07-01T23:27:54+09:00", "pid": 29342, "process": {"exists": false, "pid": 29342}, "restart_count": 38, "restart_count_per_hour": 11, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.688, "sample_count": 2176, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 291484, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:31:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T23:31:39+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T23:31:39+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-01T23:31:39.681383+09:00", "mtime_epoch": 1782916299.6813831, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91799}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:31:39+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1006.023, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.335, "pushed": 1.335, "read": 1.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.852, "count": 40, "max_ms": 5.701, "p95_ms": 5.653}, "gstreamer_push": {"avg_ms": 1.799, "count": 40, "max_ms": 7.072, "p95_ms": 4.315}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.966, "count": 40, "max_ms": 78.743, "p95_ms": 74.004}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.387, "count": 40, "max_ms": 79.374, "p95_ms": 78.33}, "yolo11_640_fire_11class": {"avg_ms": 71.95, "count": 40, "max_ms": 79.879, "p95_ms": 77.728}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.53, "count": 40, "max_ms": 87.343, "p95_ms": 85.082}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.173, "count": 40, "max_ms": 101.546, "p95_ms": 101.067}, "read_interval": {"avg_ms": 768.052, "count": 39, "max_ms": 16024.735, "p95_ms": 9804.317}, "triton_wait": {"avg_ms": 80.542, "count": 40, "max_ms": 87.354, "p95_ms": 85.095}, "visualize": {"avg_ms": 4.158, "count": 40, "max_ms": 6.45, "p95_ms": 6.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2429}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4148.715, "limit_mb": 8796093022207.996}, "detail": {"pid": 29494, "restart_count": 39, "start_count": 40}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 40, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:31:39+09:00", "last_error_summary": "[h264 @ 0x55b3fbc1fd80] error while decoding MB 101 17, bytestream -5 [in#0/rtsp @ 0x55b3fb28de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b3fb28de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:31:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:31:42+09:00", "pid": 29494, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.238, "name": "ffmpeg", "pid": 29494, "rss_mb": 57.238, "state": "S (sleeping)", "threads": 1}, "restart_count": 39, "restart_count_per_hour": 12, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.652, "sample_count": 2133, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 291484, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:31:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.781, "last_frame_pushed_at": "2026-07-01T23:31:39+09:00", "last_frame_read_age_sec": 4.878, "last_frame_read_at": "2026-07-01T23:31:39+09:00", "mjpeg": {"age_sec": 4.767, "exists": true, "mtime": "2026-07-01T23:31:39.793387+09:00", "mtime_epoch": 1782916299.7933867, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92235}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:31:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1006.023, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.588, "pushed": 0.588, "read": 0.588}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.392, "count": 13, "max_ms": 3.676, "p95_ms": 3.676}, "gstreamer_push": {"avg_ms": 1.71, "count": 13, "max_ms": 4.023, "p95_ms": 4.023}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.209, "count": 13, "max_ms": 78.743, "p95_ms": 78.743}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.458, "count": 13, "max_ms": 79.374, "p95_ms": 79.374}, "yolo11_640_fire_11class": {"avg_ms": 73.559, "count": 13, "max_ms": 79.879, "p95_ms": 79.879}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.574, "count": 13, "max_ms": 87.343, "p95_ms": 87.343}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.858, "count": 13, "max_ms": 99.291, "p95_ms": 99.291}, "read_interval": {"avg_ms": 2080.471, "count": 13, "max_ms": 16024.735, "p95_ms": 16024.735}, "triton_wait": {"avg_ms": 82.586, "count": 13, "max_ms": 87.354, "p95_ms": 87.354}, "visualize": {"avg_ms": 3.63, "count": 13, "max_ms": 3.819, "p95_ms": 3.819}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.12, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1521}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4177.262, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55deef556580] top block unavailable for requested intra mode [h264 @ 0x55deef556580] error while decoding MB 21 0, bytestream 6207 [h264 @ 0x55def0073ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55def0073ec0] error while decoding MB 0 13, bytestream 5242 [h264 @ 0x55deef4d30c0] top block unavailable for requested intra mode [h264 @ 0x55deef4d30c0] error while decoding MB 48 0, bytestream 7341 [h264 @ 0x55deef57e780] left block unavailable for requested intra mod", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 172, "h264_decode_error_count": 239, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:34:12+09:00", "last_error_summary": "[h264 @ 0x55deef556580] top block unavailable for requested intra mode [h264 @ 0x55deef556580] error while decoding MB 21 0, bytestream 6207 [h264 @ 0x55def0073ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55def0073ec0] error while decoding MB 0 13, bytestream 5242 [h264 @ 0x55deef4d30c0] top block unavailable for requested intra mode [h264 @ 0x55deef4d30c0] error while decoding MB 48 0, bytestream 7341 [h264 @ 0x55deef57e780] left block unavailable for requested intra mod", "last_exit_at": "2026-07-01T23:22:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 677.46, "last_started_at": "2026-07-01T23:22:55+09:00", "pid": 29083, "process": {"exists": false, "pid": 29083}, "restart_count": 196, "restart_count_per_hour": 5, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 197}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.188, "sample_count": 2686, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182552, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:34:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-01T23:34:12+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-01T23:34:12+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-01T23:34:12.901962+09:00", "mtime_epoch": 1782916452.9019623, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116105}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:34:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 954.086, "name": "celery", "pid": 237, "rss_mb": 948.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.3, "pushed": 3.3, "read": 3.333}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.543, "count": 99, "max_ms": 1.902, "p95_ms": 1.573}, "gstreamer_push": {"avg_ms": 1.565, "count": 99, "max_ms": 4.11, "p95_ms": 3.888}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.569, "count": 99, "max_ms": 78.472, "p95_ms": 77.042}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.172, "count": 99, "max_ms": 83.071, "p95_ms": 79.951}, "yolo11_640_fire_11class": {"avg_ms": 74.077, "count": 99, "max_ms": 78.562, "p95_ms": 77.11}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.928, "count": 99, "max_ms": 87.302, "p95_ms": 86.345}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.432, "count": 99, "max_ms": 97.738, "p95_ms": 96.801}, "read_interval": {"avg_ms": 300.565, "count": 100, "max_ms": 19161.983, "p95_ms": 114.629}, "triton_wait": {"avg_ms": 82.968, "count": 99, "max_ms": 87.312, "p95_ms": 86.357}, "visualize": {"avg_ms": 3.689, "count": 99, "max_ms": 4.441, "p95_ms": 3.841}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1521}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4176.715, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:34:12+09:00", "last_error_summary": "[h264 @ 0x55deef556580] top block unavailable for requested intra mode [h264 @ 0x55deef556580] error while decoding MB 21 0, bytestream 6207 [h264 @ 0x55def0073ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55def0073ec0] error while decoding MB 0 13, bytestream 5242 [h264 @ 0x55deef4d30c0] top block unavailable for requested intra mode [h264 @ 0x55deef4d30c0] error while decoding MB 48 0, bytestream 7341 [h264 @ 0x55deef57e780] left block unavailable for requested intra mod", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 172, "h264_decode_error_count": 239, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:34:12+09:00", "last_error_summary": "[h264 @ 0x55deef556580] top block unavailable for requested intra mode [h264 @ 0x55deef556580] error while decoding MB 21 0, bytestream 6207 [h264 @ 0x55def0073ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55def0073ec0] error while decoding MB 0 13, bytestream 5242 [h264 @ 0x55deef4d30c0] top block unavailable for requested intra mode [h264 @ 0x55deef4d30c0] error while decoding MB 48 0, bytestream 7341 [h264 @ 0x55deef57e780] left block unavailable for requested intra mod", "last_exit_at": "2026-07-01T23:34:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 677.502, "last_started_at": "2026-07-01T23:22:55+09:00", "pid": 29083, "process": {"exists": false, "pid": 29083}, "restart_count": 196, "restart_count_per_hour": 5, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 197}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.188, "sample_count": 2687, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182553, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:34:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T23:34:12+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-01T23:34:12+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-01T23:34:12.901962+09:00", "mtime_epoch": 1782916452.9019623, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116105}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:34:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 954.086, "name": "celery", "pid": 237, "rss_mb": 948.672, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.339, "pushed": 3.339, "read": 3.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.543, "count": 100, "max_ms": 1.902, "p95_ms": 1.571}, "gstreamer_push": {"avg_ms": 1.562, "count": 100, "max_ms": 4.11, "p95_ms": 3.735}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.446, "count": 100, "max_ms": 78.472, "p95_ms": 77.022}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.196, "count": 100, "max_ms": 83.071, "p95_ms": 79.332}, "yolo11_640_fire_11class": {"avg_ms": 74.146, "count": 100, "max_ms": 81.047, "p95_ms": 77.11}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.752, "count": 100, "max_ms": 87.302, "p95_ms": 86.31}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.408, "count": 100, "max_ms": 97.738, "p95_ms": 96.692}, "read_interval": {"avg_ms": 302.44, "count": 99, "max_ms": 19161.983, "p95_ms": 114.629}, "triton_wait": {"avg_ms": 82.949, "count": 100, "max_ms": 87.312, "p95_ms": 86.32}, "visualize": {"avg_ms": 3.687, "count": 100, "max_ms": 4.441, "p95_ms": 3.819}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1521}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4176.516, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 172, "h264_decode_error_count": 239, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:34:12+09:00", "last_error_summary": "[h264 @ 0x55deef556580] top block unavailable for requested intra mode [h264 @ 0x55deef556580] error while decoding MB 21 0, bytestream 6207 [h264 @ 0x55def0073ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55def0073ec0] error while decoding MB 0 13, bytestream 5242 [h264 @ 0x55deef4d30c0] top block unavailable for requested intra mode [h264 @ 0x55deef4d30c0] error while decoding MB 48 0, bytestream 7341 [h264 @ 0x55deef57e780] left block unavailable for requested intra mod", "last_exit_at": "2026-07-01T23:34:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 677.51, "last_started_at": "2026-07-01T23:22:55+09:00", "pid": 29083, "process": {"exists": false, "pid": 29083}, "restart_count": 196, "restart_count_per_hour": 5, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 197}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.188, "sample_count": 2687, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182553, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:34:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T23:34:12+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-01T23:34:12+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-01T23:34:12.901962+09:00", "mtime_epoch": 1782916452.9019623, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116105}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:34:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 954.086, "name": "celery", "pid": 237, "rss_mb": 948.672, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.338, "pushed": 3.338, "read": 3.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.543, "count": 100, "max_ms": 1.902, "p95_ms": 1.571}, "gstreamer_push": {"avg_ms": 1.562, "count": 100, "max_ms": 4.11, "p95_ms": 3.735}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.446, "count": 100, "max_ms": 78.472, "p95_ms": 77.022}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.196, "count": 100, "max_ms": 83.071, "p95_ms": 79.332}, "yolo11_640_fire_11class": {"avg_ms": 74.146, "count": 100, "max_ms": 81.047, "p95_ms": 77.11}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.752, "count": 100, "max_ms": 87.302, "p95_ms": 86.31}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.408, "count": 100, "max_ms": 97.738, "p95_ms": 96.692}, "read_interval": {"avg_ms": 302.44, "count": 99, "max_ms": 19161.983, "p95_ms": 114.629}, "triton_wait": {"avg_ms": 82.949, "count": 100, "max_ms": 87.312, "p95_ms": 86.32}, "visualize": {"avg_ms": 3.687, "count": 100, "max_ms": 4.441, "p95_ms": 3.819}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1521}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4284.391, "limit_mb": 8796093022207.996}, "detail": {"pid": 29646, "restart_count": 197, "start_count": 198}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 172, "h264_decode_error_count": 239, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:34:12+09:00", "last_error_summary": "[h264 @ 0x55deef556580] top block unavailable for requested intra mode [h264 @ 0x55deef556580] error while decoding MB 21 0, bytestream 6207 [h264 @ 0x55def0073ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55def0073ec0] error while decoding MB 0 13, bytestream 5242 [h264 @ 0x55deef4d30c0] top block unavailable for requested intra mode [h264 @ 0x55deef4d30c0] error while decoding MB 48 0, bytestream 7341 [h264 @ 0x55deef57e780] left block unavailable for requested intra mod", "last_exit_at": "2026-07-01T23:34:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:34:15+09:00", "pid": 29646, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.207, "name": "ffmpeg", "pid": 29646, "rss_mb": 139.207, "state": "S (sleeping)", "threads": 106}, "restart_count": 197, "restart_count_per_hour": 6, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 198}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.148, "sample_count": 2649, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 182553, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:34:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.278, "last_frame_pushed_at": "2026-07-01T23:34:12+09:00", "last_frame_read_age_sec": 4.371, "last_frame_read_at": "2026-07-01T23:34:12+09:00", "mjpeg": {"age_sec": 4.263, "exists": true, "mtime": "2026-07-01T23:34:13.005965+09:00", "mtime_epoch": 1782916453.0059652, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115845}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:34:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 954.086, "name": "celery", "pid": 237, "rss_mb": 948.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 61, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.034, "pushed": 2.034, "read": 2.034}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.542, "count": 61, "max_ms": 1.902, "p95_ms": 1.571}, "gstreamer_push": {"avg_ms": 1.467, "count": 61, "max_ms": 3.929, "p95_ms": 1.467}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.76, "count": 61, "max_ms": 77.418, "p95_ms": 76.198}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.346, "count": 61, "max_ms": 83.071, "p95_ms": 80.72}, "yolo11_640_fire_11class": {"avg_ms": 73.819, "count": 61, "max_ms": 81.047, "p95_ms": 76.535}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.223, "count": 61, "max_ms": 87.302, "p95_ms": 85.869}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.023, "count": 61, "max_ms": 97.738, "p95_ms": 96.55}, "read_interval": {"avg_ms": 421.852, "count": 61, "max_ms": 19161.983, "p95_ms": 113.673}, "triton_wait": {"avg_ms": 82.538, "count": 61, "max_ms": 87.312, "p95_ms": 85.882}, "visualize": {"avg_ms": 3.682, "count": 61, "max_ms": 3.863, "p95_ms": 3.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2448}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4117.395, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557d7660ebc0] error while decoding MB 66 53, bytestream -9 [in#0/rtsp @ 0x557d7648ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557d7648ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:36:24+09:00", "last_error_summary": "[h264 @ 0x557d7660ebc0] error while decoding MB 66 53, bytestream -9 [in#0/rtsp @ 0x557d7648ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557d7648ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:31:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 281.649, "last_started_at": "2026-07-01T23:31:42+09:00", "pid": 29494, "process": {"exists": false, "pid": 29494}, "restart_count": 39, "restart_count_per_hour": 11, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.876, "sample_count": 2370, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 293842, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:36:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-01T23:36:24+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-01T23:36:24+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-01T23:36:24.145555+09:00", "mtime_epoch": 1782916584.1455545, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95260}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:36:24+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1009.633, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.685, "pushed": 0.685, "read": 0.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.706, "count": 12, "max_ms": 5.05, "p95_ms": 5.05}, "gstreamer_push": {"avg_ms": 1.945, "count": 12, "max_ms": 3.922, "p95_ms": 3.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.861, "count": 12, "max_ms": 70.97, "p95_ms": 70.97}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.298, "count": 12, "max_ms": 74.276, "p95_ms": 74.276}, "yolo11_640_fire_11class": {"avg_ms": 65.035, "count": 12, "max_ms": 71.292, "p95_ms": 71.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.409, "count": 12, "max_ms": 83.572, "p95_ms": 83.572}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.208, "count": 12, "max_ms": 95.861, "p95_ms": 95.861}, "read_interval": {"avg_ms": 2533.424, "count": 13, "max_ms": 16265.48, "p95_ms": 16265.48}, "triton_wait": {"avg_ms": 76.462, "count": 12, "max_ms": 83.584, "p95_ms": 83.584}, "visualize": {"avg_ms": 3.624, "count": 12, "max_ms": 3.761, "p95_ms": 3.761}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.51, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2448}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4114.312, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:36:24+09:00", "last_error_summary": "[h264 @ 0x557d7660ebc0] error while decoding MB 66 53, bytestream -9 [in#0/rtsp @ 0x557d7648ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557d7648ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:36:24+09:00", "last_error_summary": "[h264 @ 0x557d7660ebc0] error while decoding MB 66 53, bytestream -9 [in#0/rtsp @ 0x557d7648ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557d7648ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:36:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 281.685, "last_started_at": "2026-07-01T23:31:42+09:00", "pid": 29494, "process": {"exists": false, "pid": 29494}, "restart_count": 39, "restart_count_per_hour": 11, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.876, "sample_count": 2371, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 293843, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:36:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T23:36:24+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T23:36:24+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-01T23:36:24.145555+09:00", "mtime_epoch": 1782916584.1455545, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95260}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:36:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1009.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.741, "pushed": 0.741, "read": 0.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.675, "count": 13, "max_ms": 5.05, "p95_ms": 5.05}, "gstreamer_push": {"avg_ms": 1.896, "count": 13, "max_ms": 3.922, "p95_ms": 3.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.658, "count": 13, "max_ms": 72.225, "p95_ms": 72.225}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.588, "count": 13, "max_ms": 74.276, "p95_ms": 74.276}, "yolo11_640_fire_11class": {"avg_ms": 65.356, "count": 13, "max_ms": 71.292, "p95_ms": 71.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.669, "count": 13, "max_ms": 83.572, "p95_ms": 83.572}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.408, "count": 13, "max_ms": 95.861, "p95_ms": 95.861}, "read_interval": {"avg_ms": 2533.424, "count": 13, "max_ms": 16265.48, "p95_ms": 16265.48}, "triton_wait": {"avg_ms": 76.719, "count": 13, "max_ms": 83.584, "p95_ms": 83.584}, "visualize": {"avg_ms": 3.629, "count": 13, "max_ms": 3.761, "p95_ms": 3.761}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.545, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2448}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4114.16, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:36:24+09:00", "last_error_summary": "[h264 @ 0x557d7660ebc0] error while decoding MB 66 53, bytestream -9 [in#0/rtsp @ 0x557d7648ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557d7648ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:36:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 281.693, "last_started_at": "2026-07-01T23:31:42+09:00", "pid": 29494, "process": {"exists": false, "pid": 29494}, "restart_count": 39, "restart_count_per_hour": 11, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.876, "sample_count": 2371, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 293843, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:36:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T23:36:24+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-01T23:36:24+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T23:36:24.145555+09:00", "mtime_epoch": 1782916584.1455545, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95260}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:36:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1009.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.741, "pushed": 0.741, "read": 0.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.675, "count": 13, "max_ms": 5.05, "p95_ms": 5.05}, "gstreamer_push": {"avg_ms": 1.896, "count": 13, "max_ms": 3.922, "p95_ms": 3.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.658, "count": 13, "max_ms": 72.225, "p95_ms": 72.225}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.588, "count": 13, "max_ms": 74.276, "p95_ms": 74.276}, "yolo11_640_fire_11class": {"avg_ms": 65.356, "count": 13, "max_ms": 71.292, "p95_ms": 71.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.669, "count": 13, "max_ms": 83.572, "p95_ms": 83.572}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.408, "count": 13, "max_ms": 95.861, "p95_ms": 95.861}, "read_interval": {"avg_ms": 2533.424, "count": 13, "max_ms": 16265.48, "p95_ms": 16265.48}, "triton_wait": {"avg_ms": 76.719, "count": 13, "max_ms": 83.584, "p95_ms": 83.584}, "visualize": {"avg_ms": 3.629, "count": 13, "max_ms": 3.761, "p95_ms": 3.761}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.553, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2448}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4137.016, "limit_mb": 8796093022207.996}, "detail": {"pid": 29753, "restart_count": 40, "start_count": 41}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 41, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:36:24+09:00", "last_error_summary": "[h264 @ 0x557d7660ebc0] error while decoding MB 66 53, bytestream -9 [in#0/rtsp @ 0x557d7648ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557d7648ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:36:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:36:26+09:00", "pid": 29753, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.641, "name": "ffmpeg", "pid": 29753, "rss_mb": 57.641, "state": "S (sleeping)", "threads": 1}, "restart_count": 40, "restart_count_per_hour": 12, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.876, "sample_count": 2371, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 293843, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:36:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.266, "last_frame_pushed_at": "2026-07-01T23:36:24+09:00", "last_frame_read_age_sec": 4.36, "last_frame_read_at": "2026-07-01T23:36:24+09:00", "mjpeg": {"age_sec": 4.251, "exists": true, "mtime": "2026-07-01T23:36:24.253557+09:00", "mtime_epoch": 1782916584.2535574, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95606}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:36:28+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1009.633, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.596, "pushed": 0.596, "read": 0.596}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.675, "count": 13, "max_ms": 5.05, "p95_ms": 5.05}, "gstreamer_push": {"avg_ms": 1.896, "count": 13, "max_ms": 3.922, "p95_ms": 3.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.658, "count": 13, "max_ms": 72.225, "p95_ms": 72.225}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.588, "count": 13, "max_ms": 74.276, "p95_ms": 74.276}, "yolo11_640_fire_11class": {"avg_ms": 65.356, "count": 13, "max_ms": 71.292, "p95_ms": 71.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.669, "count": 13, "max_ms": 83.572, "p95_ms": 83.572}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.408, "count": 13, "max_ms": 95.861, "p95_ms": 95.861}, "read_interval": {"avg_ms": 2533.424, "count": 13, "max_ms": 16265.48, "p95_ms": 16265.48}, "triton_wait": {"avg_ms": 76.719, "count": 13, "max_ms": 83.584, "p95_ms": 83.584}, "visualize": {"avg_ms": 3.629, "count": 13, "max_ms": 3.761, "p95_ms": 3.761}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.805, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4118.797, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55aaac8df200] missing picture in access unit with size 18 [h264 @ 0x55aaad006dc0] no frame! [in#0/rtsp @ 0x55aaac8dae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aaac8dae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55aaac8df840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:38:54+09:00", "last_error_summary": "[NULL @ 0x55aaac8df200] missing picture in access unit with size 18 [h264 @ 0x55aaad006dc0] no frame! [in#0/rtsp @ 0x55aaac8dae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aaac8dae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55aaac8df840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:36:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.481, "last_started_at": "2026-07-01T23:36:26+09:00", "pid": 29753, "process": {"exists": false, "pid": 29753}, "restart_count": 40, "restart_count_per_hour": 11, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.486, "sample_count": 2202, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 294880, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:38:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.086, "last_frame_pushed_at": "2026-07-01T23:38:54+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-01T23:38:54+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-01T23:38:54.913398+09:00", "mtime_epoch": 1782916734.913398, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:38:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1007.57, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.693, "pushed": 0.693, "read": 0.751}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.414, "count": 12, "max_ms": 3.517, "p95_ms": 3.517}, "gstreamer_push": {"avg_ms": 1.362, "count": 12, "max_ms": 1.444, "p95_ms": 1.444}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.793, "count": 12, "max_ms": 70.799, "p95_ms": 70.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.012, "count": 12, "max_ms": 77.433, "p95_ms": 77.433}, "yolo11_640_fire_11class": {"avg_ms": 64.381, "count": 12, "max_ms": 76.597, "p95_ms": 76.597}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.837, "count": 12, "max_ms": 83.596, "p95_ms": 83.596}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.313, "count": 12, "max_ms": 96.039, "p95_ms": 96.039}, "read_interval": {"avg_ms": 2622.252, "count": 13, "max_ms": 16843.608, "p95_ms": 16843.608}, "triton_wait": {"avg_ms": 71.061, "count": 12, "max_ms": 83.609, "p95_ms": 83.609}, "visualize": {"avg_ms": 3.672, "count": 12, "max_ms": 3.817, "p95_ms": 3.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.318, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4116.004, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:38:54+09:00", "last_error_summary": "[NULL @ 0x55aaac8df200] missing picture in access unit with size 18 [h264 @ 0x55aaad006dc0] no frame! [in#0/rtsp @ 0x55aaac8dae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aaac8dae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55aaac8df840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:38:54+09:00", "last_error_summary": "[NULL @ 0x55aaac8df200] missing picture in access unit with size 18 [h264 @ 0x55aaad006dc0] no frame! [in#0/rtsp @ 0x55aaac8dae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aaac8dae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55aaac8df840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:38:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.518, "last_started_at": "2026-07-01T23:36:26+09:00", "pid": 29753, "process": {"exists": false, "pid": 29753}, "restart_count": 40, "restart_count_per_hour": 11, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.501, "sample_count": 2202, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 294881, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:38:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T23:38:55+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-01T23:38:54+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-01T23:38:54.913398+09:00", "mtime_epoch": 1782916734.913398, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:38:55+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.734, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.749, "pushed": 0.749, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.406, "count": 13, "max_ms": 3.517, "p95_ms": 3.517}, "gstreamer_push": {"avg_ms": 1.721, "count": 13, "max_ms": 6.032, "p95_ms": 6.032}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.952, "count": 13, "max_ms": 70.799, "p95_ms": 70.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.764, "count": 13, "max_ms": 77.433, "p95_ms": 77.433}, "yolo11_640_fire_11class": {"avg_ms": 65.19, "count": 13, "max_ms": 76.597, "p95_ms": 76.597}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.838, "count": 13, "max_ms": 83.857, "p95_ms": 83.857}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.413, "count": 13, "max_ms": 97.614, "p95_ms": 97.614}, "read_interval": {"avg_ms": 2622.252, "count": 13, "max_ms": 16843.608, "p95_ms": 16843.608}, "triton_wait": {"avg_ms": 72.046, "count": 13, "max_ms": 83.869, "p95_ms": 83.869}, "visualize": {"avg_ms": 3.671, "count": 13, "max_ms": 3.817, "p95_ms": 3.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.355, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4115.855, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:38:54+09:00", "last_error_summary": "[NULL @ 0x55aaac8df200] missing picture in access unit with size 18 [h264 @ 0x55aaad006dc0] no frame! [in#0/rtsp @ 0x55aaac8dae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aaac8dae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55aaac8df840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:38:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.525, "last_started_at": "2026-07-01T23:36:26+09:00", "pid": 29753, "process": {"exists": false, "pid": 29753}, "restart_count": 40, "restart_count_per_hour": 11, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.501, "sample_count": 2202, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 294881, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:38:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T23:38:55+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-01T23:38:54+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-01T23:38:54.913398+09:00", "mtime_epoch": 1782916734.913398, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93628}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:38:55+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.734, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.749, "pushed": 0.749, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.406, "count": 13, "max_ms": 3.517, "p95_ms": 3.517}, "gstreamer_push": {"avg_ms": 1.721, "count": 13, "max_ms": 6.032, "p95_ms": 6.032}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.952, "count": 13, "max_ms": 70.799, "p95_ms": 70.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.764, "count": 13, "max_ms": 77.433, "p95_ms": 77.433}, "yolo11_640_fire_11class": {"avg_ms": 65.19, "count": 13, "max_ms": 76.597, "p95_ms": 76.597}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.838, "count": 13, "max_ms": 83.857, "p95_ms": 83.857}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.413, "count": 13, "max_ms": 97.614, "p95_ms": 97.614}, "read_interval": {"avg_ms": 2622.252, "count": 13, "max_ms": 16843.608, "p95_ms": 16843.608}, "triton_wait": {"avg_ms": 72.046, "count": 13, "max_ms": 83.869, "p95_ms": 83.869}, "visualize": {"avg_ms": 3.671, "count": 13, "max_ms": 3.817, "p95_ms": 3.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.362, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4289.484, "limit_mb": 8796093022207.996}, "detail": {"pid": 29905, "restart_count": 41, "start_count": 42}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 42, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:38:54+09:00", "last_error_summary": "[NULL @ 0x55aaac8df200] missing picture in access unit with size 18 [h264 @ 0x55aaad006dc0] no frame! [in#0/rtsp @ 0x55aaac8dae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55aaac8dae00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55aaac8df840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-01T23:38:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:38:57+09:00", "pid": 29905, "process": {"exists": true, "fd_count": 5, "hwm_mb": 208.906, "name": "ffmpeg", "pid": 29905, "rss_mb": 208.906, "state": "S (sleeping)", "threads": 106}, "restart_count": 41, "restart_count_per_hour": 12, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.524, "sample_count": 2148, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 294881, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:38:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.2, "last_frame_pushed_at": "2026-07-01T23:38:55+09:00", "last_frame_read_age_sec": 4.304, "last_frame_read_at": "2026-07-01T23:38:54+09:00", "mjpeg": {"age_sec": 4.187, "exists": true, "mtime": "2026-07-01T23:38:55.029401+09:00", "mtime_epoch": 1782916735.0294008, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 91104}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:38:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.734, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.603, "pushed": 0.603, "read": 0.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.406, "count": 13, "max_ms": 3.517, "p95_ms": 3.517}, "gstreamer_push": {"avg_ms": 1.721, "count": 13, "max_ms": 6.032, "p95_ms": 6.032}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.952, "count": 13, "max_ms": 70.799, "p95_ms": 70.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.764, "count": 13, "max_ms": 77.433, "p95_ms": 77.433}, "yolo11_640_fire_11class": {"avg_ms": 65.19, "count": 13, "max_ms": 76.597, "p95_ms": 76.597}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.838, "count": 13, "max_ms": 83.857, "p95_ms": 83.857}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.413, "count": 13, "max_ms": 97.614, "p95_ms": 97.614}, "read_interval": {"avg_ms": 2622.252, "count": 13, "max_ms": 16843.608, "p95_ms": 16843.608}, "triton_wait": {"avg_ms": 72.046, "count": 13, "max_ms": 83.869, "p95_ms": 83.869}, "visualize": {"avg_ms": 3.671, "count": 13, "max_ms": 3.817, "p95_ms": 3.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.55, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2464}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4114.91, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562b9e681840] error while decoding MB 117 17, bytestream -5 [in#0/rtsp @ 0x562b9e021e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b9e021e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:41:11+09:00", "last_error_summary": "[h264 @ 0x562b9e681840] error while decoding MB 117 17, bytestream -5 [in#0/rtsp @ 0x562b9e021e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b9e021e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:38:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.106, "last_started_at": "2026-07-01T23:38:57+09:00", "pid": 29905, "process": {"exists": false, "pid": 29905}, "restart_count": 41, "restart_count_per_hour": 12, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.75, "sample_count": 1948, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295779, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:41:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-01T23:41:11+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-01T23:41:11+09:00", "mjpeg": {"age_sec": 0.065, "exists": true, "mtime": "2026-07-01T23:41:11.256675+09:00", "mtime_epoch": 1782916871.2566752, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94691}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:41:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.992, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.68, "pushed": 0.68, "read": 0.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.41, "count": 12, "max_ms": 3.475, "p95_ms": 3.475}, "gstreamer_push": {"avg_ms": 1.337, "count": 12, "max_ms": 1.423, "p95_ms": 1.423}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.26, "count": 12, "max_ms": 66.473, "p95_ms": 66.473}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.444, "count": 12, "max_ms": 79.518, "p95_ms": 79.518}, "yolo11_640_fire_11class": {"avg_ms": 70.432, "count": 12, "max_ms": 76.869, "p95_ms": 76.869}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.407, "count": 12, "max_ms": 85.27, "p95_ms": 85.27}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.769, "count": 12, "max_ms": 97.431, "p95_ms": 97.431}, "read_interval": {"avg_ms": 2600.342, "count": 13, "max_ms": 16399.478, "p95_ms": 16399.478}, "triton_wait": {"avg_ms": 79.708, "count": 12, "max_ms": 85.282, "p95_ms": 85.282}, "visualize": {"avg_ms": 3.637, "count": 12, "max_ms": 3.774, "p95_ms": 3.774}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.655, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2464}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4114.164, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:41:11+09:00", "last_error_summary": "[h264 @ 0x562b9e681840] error while decoding MB 117 17, bytestream -5 [in#0/rtsp @ 0x562b9e021e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b9e021e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:41:11+09:00", "last_error_summary": "[h264 @ 0x562b9e681840] error while decoding MB 117 17, bytestream -5 [in#0/rtsp @ 0x562b9e021e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b9e021e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:41:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.148, "last_started_at": "2026-07-01T23:38:57+09:00", "pid": 29905, "process": {"exists": false, "pid": 29905}, "restart_count": 41, "restart_count_per_hour": 12, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.791, "sample_count": 1949, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295780, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:41:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-01T23:41:11+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-01T23:41:11+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-01T23:41:11.256675+09:00", "mtime_epoch": 1782916871.2566752, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94691}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:41:11+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.992, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.735, "pushed": 0.735, "read": 0.735}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.404, "count": 13, "max_ms": 3.475, "p95_ms": 3.475}, "gstreamer_push": {"avg_ms": 1.332, "count": 13, "max_ms": 1.423, "p95_ms": 1.423}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.514, "count": 13, "max_ms": 67.567, "p95_ms": 67.567}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.787, "count": 13, "max_ms": 79.518, "p95_ms": 79.518}, "yolo11_640_fire_11class": {"avg_ms": 70.894, "count": 13, "max_ms": 76.869, "p95_ms": 76.869}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.007, "count": 13, "max_ms": 87.205, "p95_ms": 87.205}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.345, "count": 13, "max_ms": 99.255, "p95_ms": 99.255}, "read_interval": {"avg_ms": 2600.342, "count": 13, "max_ms": 16399.478, "p95_ms": 16399.478}, "triton_wait": {"avg_ms": 80.285, "count": 13, "max_ms": 87.217, "p95_ms": 87.217}, "visualize": {"avg_ms": 3.642, "count": 13, "max_ms": 3.774, "p95_ms": 3.774}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.696, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2464}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4114.055, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:41:11+09:00", "last_error_summary": "[h264 @ 0x562b9e681840] error while decoding MB 117 17, bytestream -5 [in#0/rtsp @ 0x562b9e021e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b9e021e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:41:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.155, "last_started_at": "2026-07-01T23:38:57+09:00", "pid": 29905, "process": {"exists": false, "pid": 29905}, "restart_count": 41, "restart_count_per_hour": 12, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.791, "sample_count": 1949, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295780, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:41:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-01T23:41:11+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-01T23:41:11+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-01T23:41:11.256675+09:00", "mtime_epoch": 1782916871.2566752, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 94691}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:41:11+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.992, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.734, "pushed": 0.734, "read": 0.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.404, "count": 13, "max_ms": 3.475, "p95_ms": 3.475}, "gstreamer_push": {"avg_ms": 1.332, "count": 13, "max_ms": 1.423, "p95_ms": 1.423}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.514, "count": 13, "max_ms": 67.567, "p95_ms": 67.567}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.787, "count": 13, "max_ms": 79.518, "p95_ms": 79.518}, "yolo11_640_fire_11class": {"avg_ms": 70.894, "count": 13, "max_ms": 76.869, "p95_ms": 76.869}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.007, "count": 13, "max_ms": 87.205, "p95_ms": 87.205}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.345, "count": 13, "max_ms": 99.255, "p95_ms": 99.255}, "read_interval": {"avg_ms": 2600.342, "count": 13, "max_ms": 16399.478, "p95_ms": 16399.478}, "triton_wait": {"avg_ms": 80.285, "count": 13, "max_ms": 87.217, "p95_ms": 87.217}, "visualize": {"avg_ms": 3.642, "count": 13, "max_ms": 3.774, "p95_ms": 3.774}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.703, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2464}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4286.023, "limit_mb": 8796093022207.996}, "detail": {"pid": 30057, "restart_count": 42, "start_count": 43}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:41:11+09:00", "last_error_summary": "[h264 @ 0x562b9e681840] error while decoding MB 117 17, bytestream -5 [in#0/rtsp @ 0x562b9e021e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562b9e021e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:41:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:41:14+09:00", "pid": 30057, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.672, "name": "ffmpeg", "pid": 30057, "rss_mb": 206.672, "state": "S (sleeping)", "threads": 106}, "restart_count": 42, "restart_count_per_hour": 13, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.791, "sample_count": 1949, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 295780, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:41:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.762, "last_frame_pushed_at": "2026-07-01T23:41:11+09:00", "last_frame_read_age_sec": 4.863, "last_frame_read_at": "2026-07-01T23:41:11+09:00", "mjpeg": {"age_sec": 4.748, "exists": true, "mtime": "2026-07-01T23:41:11.372678+09:00", "mtime_epoch": 1782916871.3726778, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95494}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:41:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1004.992, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.579, "pushed": 0.579, "read": 0.579}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.404, "count": 13, "max_ms": 3.475, "p95_ms": 3.475}, "gstreamer_push": {"avg_ms": 1.332, "count": 13, "max_ms": 1.423, "p95_ms": 1.423}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.514, "count": 13, "max_ms": 67.567, "p95_ms": 67.567}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.787, "count": 13, "max_ms": 79.518, "p95_ms": 79.518}, "yolo11_640_fire_11class": {"avg_ms": 70.894, "count": 13, "max_ms": 76.869, "p95_ms": 76.869}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.007, "count": 13, "max_ms": 87.205, "p95_ms": 87.205}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.345, "count": 13, "max_ms": 99.255, "p95_ms": 99.255}, "read_interval": {"avg_ms": 2600.342, "count": 13, "max_ms": 16399.478, "p95_ms": 16399.478}, "triton_wait": {"avg_ms": 80.285, "count": 13, "max_ms": 87.217, "p95_ms": 87.217}, "visualize": {"avg_ms": 3.642, "count": 13, "max_ms": 3.774, "p95_ms": 3.774}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.453, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2473}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4127.066, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5643ad5ae9c0] error while decoding MB 53 41, bytestream -5 [in#0/rtsp @ 0x5643acc19e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5643acc19e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:43:43+09:00", "last_error_summary": "[h264 @ 0x5643ad5ae9c0] error while decoding MB 53 41, bytestream -5 [in#0/rtsp @ 0x5643acc19e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5643acc19e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:41:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.524, "last_started_at": "2026-07-01T23:41:14+09:00", "pid": 30057, "process": {"exists": false, "pid": 30057}, "restart_count": 42, "restart_count_per_hour": 12, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.276, "sample_count": 1962, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 296831, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:43:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-01T23:43:43+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-01T23:43:43+09:00", "mjpeg": {"age_sec": 0.067, "exists": true, "mtime": "2026-07-01T23:43:43.576169+09:00", "mtime_epoch": 1782917023.5761688, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90920}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:43:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1008.344, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.75, "pushed": 0.75, "read": 0.75}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.303, "count": 13, "max_ms": 3.509, "p95_ms": 3.509}, "gstreamer_push": {"avg_ms": 1.53, "count": 13, "max_ms": 3.843, "p95_ms": 3.843}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.043, "count": 13, "max_ms": 51.564, "p95_ms": 51.564}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.011, "count": 13, "max_ms": 64.271, "p95_ms": 64.271}, "yolo11_640_fire_11class": {"avg_ms": 55.247, "count": 13, "max_ms": 65.889, "p95_ms": 65.889}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.804, "count": 13, "max_ms": 64.79, "p95_ms": 64.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.654, "count": 13, "max_ms": 77.855, "p95_ms": 77.855}, "read_interval": {"avg_ms": 2587.171, "count": 13, "max_ms": 16360.64, "p95_ms": 16360.64}, "triton_wait": {"avg_ms": 56.473, "count": 13, "max_ms": 65.9, "p95_ms": 65.9}, "visualize": {"avg_ms": 3.547, "count": 13, "max_ms": 3.71, "p95_ms": 3.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.34, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2473}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4127.055, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:43:43+09:00", "last_error_summary": "[h264 @ 0x5643ad5ae9c0] error while decoding MB 53 41, bytestream -5 [in#0/rtsp @ 0x5643acc19e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5643acc19e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:43:43+09:00", "last_error_summary": "[h264 @ 0x5643ad5ae9c0] error while decoding MB 53 41, bytestream -5 [in#0/rtsp @ 0x5643acc19e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5643acc19e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:43:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.532, "last_started_at": "2026-07-01T23:41:14+09:00", "pid": 30057, "process": {"exists": false, "pid": 30057}, "restart_count": 42, "restart_count_per_hour": 12, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.276, "sample_count": 1962, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 296831, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:43:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-01T23:43:43+09:00", "last_frame_read_age_sec": 0.075, "last_frame_read_at": "2026-07-01T23:43:43+09:00", "mjpeg": {"age_sec": 0.075, "exists": true, "mtime": "2026-07-01T23:43:43.576169+09:00", "mtime_epoch": 1782917023.5761688, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90920}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:43:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1008.344, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.749, "pushed": 0.749, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.303, "count": 13, "max_ms": 3.509, "p95_ms": 3.509}, "gstreamer_push": {"avg_ms": 1.53, "count": 13, "max_ms": 3.843, "p95_ms": 3.843}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.043, "count": 13, "max_ms": 51.564, "p95_ms": 51.564}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.011, "count": 13, "max_ms": 64.271, "p95_ms": 64.271}, "yolo11_640_fire_11class": {"avg_ms": 55.247, "count": 13, "max_ms": 65.889, "p95_ms": 65.889}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.804, "count": 13, "max_ms": 64.79, "p95_ms": 64.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.654, "count": 13, "max_ms": 77.855, "p95_ms": 77.855}, "read_interval": {"avg_ms": 2587.171, "count": 13, "max_ms": 16360.64, "p95_ms": 16360.64}, "triton_wait": {"avg_ms": 56.473, "count": 13, "max_ms": 65.9, "p95_ms": 65.9}, "visualize": {"avg_ms": 3.547, "count": 13, "max_ms": 3.71, "p95_ms": 3.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.347, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2473}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4126.781, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:43:43+09:00", "last_error_summary": "[h264 @ 0x5643ad5ae9c0] error while decoding MB 53 41, bytestream -5 [in#0/rtsp @ 0x5643acc19e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5643acc19e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:43:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.539, "last_started_at": "2026-07-01T23:41:14+09:00", "pid": 30057, "process": {"exists": false, "pid": 30057}, "restart_count": 42, "restart_count_per_hour": 12, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.276, "sample_count": 1962, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 296831, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:43:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-01T23:43:43+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-01T23:43:43+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-01T23:43:43.576169+09:00", "mtime_epoch": 1782917023.5761688, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 90920}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:43:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1008.344, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.749, "pushed": 0.749, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.303, "count": 13, "max_ms": 3.509, "p95_ms": 3.509}, "gstreamer_push": {"avg_ms": 1.53, "count": 13, "max_ms": 3.843, "p95_ms": 3.843}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.043, "count": 13, "max_ms": 51.564, "p95_ms": 51.564}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.011, "count": 13, "max_ms": 64.271, "p95_ms": 64.271}, "yolo11_640_fire_11class": {"avg_ms": 55.247, "count": 13, "max_ms": 65.889, "p95_ms": 65.889}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.804, "count": 13, "max_ms": 64.79, "p95_ms": 64.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.654, "count": 13, "max_ms": 77.855, "p95_ms": 77.855}, "read_interval": {"avg_ms": 2587.171, "count": 13, "max_ms": 16360.64, "p95_ms": 16360.64}, "triton_wait": {"avg_ms": 56.473, "count": 13, "max_ms": 65.9, "p95_ms": 65.9}, "visualize": {"avg_ms": 3.547, "count": 13, "max_ms": 3.71, "p95_ms": 3.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.355, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2473}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4250.527, "limit_mb": 8796093022207.996}, "detail": {"pid": 30209, "restart_count": 43, "start_count": 44}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 44, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:43:43+09:00", "last_error_summary": "[h264 @ 0x5643ad5ae9c0] error while decoding MB 53 41, bytestream -5 [in#0/rtsp @ 0x5643acc19e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5643acc19e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:43:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:43:45+09:00", "pid": 30209, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.676, "name": "ffmpeg", "pid": 30209, "rss_mb": 156.676, "state": "R (running)", "threads": 49}, "restart_count": 43, "restart_count_per_hour": 13, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.276, "sample_count": 1962, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 296831, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:43:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.192, "last_frame_pushed_at": "2026-07-01T23:43:43+09:00", "last_frame_read_age_sec": 4.256, "last_frame_read_at": "2026-07-01T23:43:43+09:00", "mjpeg": {"age_sec": 4.18, "exists": true, "mtime": "2026-07-01T23:43:43.652170+09:00", "mtime_epoch": 1782917023.6521704, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 92330}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:43:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1008.344, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.604, "pushed": 0.604, "read": 0.604}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.303, "count": 13, "max_ms": 3.509, "p95_ms": 3.509}, "gstreamer_push": {"avg_ms": 1.53, "count": 13, "max_ms": 3.843, "p95_ms": 3.843}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.043, "count": 13, "max_ms": 51.564, "p95_ms": 51.564}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.011, "count": 13, "max_ms": 64.271, "p95_ms": 64.271}, "yolo11_640_fire_11class": {"avg_ms": 55.247, "count": 13, "max_ms": 65.889, "p95_ms": 65.889}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.804, "count": 13, "max_ms": 64.79, "p95_ms": 64.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 68.654, "count": 13, "max_ms": 77.855, "p95_ms": 77.855}, "read_interval": {"avg_ms": 2587.171, "count": 13, "max_ms": 16360.64, "p95_ms": 16360.64}, "triton_wait": {"avg_ms": 56.473, "count": 13, "max_ms": 65.9, "p95_ms": 65.9}, "visualize": {"avg_ms": 3.547, "count": 13, "max_ms": 3.71, "p95_ms": 3.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.528, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1607}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4181.855, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f8244c7a00] top block unavailable for requested intra mode [h264 @ 0x55f8244c7a00] error while decoding MB 39 0, bytestream 7888 [h264 @ 0x55f82380a380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f82380a380] error while decoding MB 0 12, bytestream 13345 [h264 @ 0x55f8238f9240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f8238f9240] error while decoding MB 0 5, bytestream 14184 [h264 @ 0x55f8244c7a00] error while decoding MB 2 4, bytestrea", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 173, "h264_decode_error_count": 243, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:52:15+09:00", "last_error_summary": "[h264 @ 0x55f8244c7a00] top block unavailable for requested intra mode [h264 @ 0x55f8244c7a00] error while decoding MB 39 0, bytestream 7888 [h264 @ 0x55f82380a380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f82380a380] error while decoding MB 0 12, bytestream 13345 [h264 @ 0x55f8238f9240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f8238f9240] error while decoding MB 0 5, bytestream 14184 [h264 @ 0x55f8244c7a00] error while decoding MB 2 4, bytestrea", "last_exit_at": "2026-07-01T23:34:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 1080.511, "last_started_at": "2026-07-01T23:34:15+09:00", "pid": 29646, "process": {"exists": false, "pid": 29646}, "restart_count": 197, "restart_count_per_hour": 5, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 198}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.006, "sample_count": 2587, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 192907, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:52:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-01T23:52:15+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-01T23:52:15+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-01T23:52:15.727123+09:00", "mtime_epoch": 1782917535.7271228, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114829}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:52:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 956.109, "name": "celery", "pid": 237, "rss_mb": 951.211, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.175, "pushed": 3.175, "read": 3.175}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.536, "count": 95, "max_ms": 2.205, "p95_ms": 1.927}, "gstreamer_push": {"avg_ms": 1.529, "count": 95, "max_ms": 6.132, "p95_ms": 3.538}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.633, "count": 95, "max_ms": 79.862, "p95_ms": 77.214}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.63, "count": 95, "max_ms": 82.783, "p95_ms": 80.284}, "yolo11_640_fire_11class": {"avg_ms": 73.663, "count": 95, "max_ms": 79.944, "p95_ms": 79.627}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.618, "count": 95, "max_ms": 92.026, "p95_ms": 88.959}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.008, "count": 95, "max_ms": 102.302, "p95_ms": 99.452}, "read_interval": {"avg_ms": 315.322, "count": 95, "max_ms": 19724.314, "p95_ms": 116.894}, "triton_wait": {"avg_ms": 82.79, "count": 95, "max_ms": 92.037, "p95_ms": 88.972}, "visualize": {"avg_ms": 3.589, "count": 95, "max_ms": 4.601, "p95_ms": 3.826}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1607}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4181.453, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:52:15+09:00", "last_error_summary": "[h264 @ 0x55f8244c7a00] top block unavailable for requested intra mode [h264 @ 0x55f8244c7a00] error while decoding MB 39 0, bytestream 7888 [h264 @ 0x55f82380a380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f82380a380] error while decoding MB 0 12, bytestream 13345 [h264 @ 0x55f8238f9240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f8238f9240] error while decoding MB 0 5, bytestream 14184 [h264 @ 0x55f8244c7a00] error while decoding MB 2 4, bytestrea", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 173, "h264_decode_error_count": 243, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:52:15+09:00", "last_error_summary": "[h264 @ 0x55f8244c7a00] top block unavailable for requested intra mode [h264 @ 0x55f8244c7a00] error while decoding MB 39 0, bytestream 7888 [h264 @ 0x55f82380a380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f82380a380] error while decoding MB 0 12, bytestream 13345 [h264 @ 0x55f8238f9240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f8238f9240] error while decoding MB 0 5, bytestream 14184 [h264 @ 0x55f8244c7a00] error while decoding MB 2 4, bytestrea", "last_exit_at": "2026-07-01T23:52:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 1080.557, "last_started_at": "2026-07-01T23:34:15+09:00", "pid": 29646, "process": {"exists": false, "pid": 29646}, "restart_count": 197, "restart_count_per_hour": 5, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 198}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.006, "sample_count": 2588, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 192908, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:52:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T23:52:15+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-01T23:52:15+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-01T23:52:15.727123+09:00", "mtime_epoch": 1782917535.7271228, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114829}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:52:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 956.109, "name": "celery", "pid": 237, "rss_mb": 951.211, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.204, "pushed": 3.204, "read": 3.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.536, "count": 96, "max_ms": 2.205, "p95_ms": 1.927}, "gstreamer_push": {"avg_ms": 1.528, "count": 96, "max_ms": 6.132, "p95_ms": 3.538}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.681, "count": 96, "max_ms": 79.862, "p95_ms": 77.214}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.637, "count": 96, "max_ms": 82.783, "p95_ms": 80.284}, "yolo11_640_fire_11class": {"avg_ms": 73.667, "count": 96, "max_ms": 79.944, "p95_ms": 79.627}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.636, "count": 96, "max_ms": 92.026, "p95_ms": 88.959}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.031, "count": 96, "max_ms": 102.302, "p95_ms": 99.452}, "read_interval": {"avg_ms": 315.322, "count": 95, "max_ms": 19724.314, "p95_ms": 116.894}, "triton_wait": {"avg_ms": 82.807, "count": 96, "max_ms": 92.037, "p95_ms": 88.972}, "visualize": {"avg_ms": 3.592, "count": 96, "max_ms": 4.601, "p95_ms": 3.851}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1607}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4181.504, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 173, "h264_decode_error_count": 243, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:52:15+09:00", "last_error_summary": "[h264 @ 0x55f8244c7a00] top block unavailable for requested intra mode [h264 @ 0x55f8244c7a00] error while decoding MB 39 0, bytestream 7888 [h264 @ 0x55f82380a380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f82380a380] error while decoding MB 0 12, bytestream 13345 [h264 @ 0x55f8238f9240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f8238f9240] error while decoding MB 0 5, bytestream 14184 [h264 @ 0x55f8244c7a00] error while decoding MB 2 4, bytestrea", "last_exit_at": "2026-07-01T23:52:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 1080.564, "last_started_at": "2026-07-01T23:34:15+09:00", "pid": 29646, "process": {"exists": false, "pid": 29646}, "restart_count": 197, "restart_count_per_hour": 5, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 198}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.006, "sample_count": 2588, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 192908, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:52:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-01T23:52:15+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-01T23:52:15+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-01T23:52:15.727123+09:00", "mtime_epoch": 1782917535.7271228, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114829}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:52:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 956.109, "name": "celery", "pid": 237, "rss_mb": 951.211, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.203, "pushed": 3.203, "read": 3.17}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.536, "count": 96, "max_ms": 2.205, "p95_ms": 1.927}, "gstreamer_push": {"avg_ms": 1.528, "count": 96, "max_ms": 6.132, "p95_ms": 3.538}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.681, "count": 96, "max_ms": 79.862, "p95_ms": 77.214}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.637, "count": 96, "max_ms": 82.783, "p95_ms": 80.284}, "yolo11_640_fire_11class": {"avg_ms": 73.667, "count": 96, "max_ms": 79.944, "p95_ms": 79.627}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.636, "count": 96, "max_ms": 92.026, "p95_ms": 88.959}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.031, "count": 96, "max_ms": 102.302, "p95_ms": 99.452}, "read_interval": {"avg_ms": 315.322, "count": 95, "max_ms": 19724.314, "p95_ms": 116.894}, "triton_wait": {"avg_ms": 82.807, "count": 96, "max_ms": 92.037, "p95_ms": 88.972}, "visualize": {"avg_ms": 3.592, "count": 96, "max_ms": 4.601, "p95_ms": 3.851}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1607}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4289.121, "limit_mb": 8796093022207.996}, "detail": {"pid": 30361, "restart_count": 198, "start_count": 199}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 173, "h264_decode_error_count": 243, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:52:15+09:00", "last_error_summary": "[h264 @ 0x55f8244c7a00] top block unavailable for requested intra mode [h264 @ 0x55f8244c7a00] error while decoding MB 39 0, bytestream 7888 [h264 @ 0x55f82380a380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f82380a380] error while decoding MB 0 12, bytestream 13345 [h264 @ 0x55f8238f9240] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f8238f9240] error while decoding MB 0 5, bytestream 14184 [h264 @ 0x55f8244c7a00] error while decoding MB 2 4, bytestrea", "last_exit_at": "2026-07-01T23:52:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:52:18+09:00", "pid": 30361, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.352, "name": "ffmpeg", "pid": 30361, "rss_mb": 141.352, "state": "S (sleeping)", "threads": 106}, "restart_count": 198, "restart_count_per_hour": 6, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 199}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.0, "sample_count": 2544, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 192908, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:52:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.931, "last_frame_pushed_at": "2026-07-01T23:52:15+09:00", "last_frame_read_age_sec": 5.028, "last_frame_read_at": "2026-07-01T23:52:15+09:00", "mjpeg": {"age_sec": 4.91, "exists": true, "mtime": "2026-07-01T23:52:15.839125+09:00", "mtime_epoch": 1782917535.839125, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112868}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-01T23:52:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 956.109, "name": "celery", "pid": 237, "rss_mb": 951.211, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 51, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.738, "pushed": 1.738, "read": 1.705}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.52, "count": 52, "max_ms": 2.205, "p95_ms": 2.016}, "gstreamer_push": {"avg_ms": 1.527, "count": 52, "max_ms": 6.132, "p95_ms": 2.317}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.241, "count": 52, "max_ms": 79.862, "p95_ms": 78.695}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.872, "count": 52, "max_ms": 82.783, "p95_ms": 80.385}, "yolo11_640_fire_11class": {"avg_ms": 72.733, "count": 52, "max_ms": 79.944, "p95_ms": 79.655}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.957, "count": 52, "max_ms": 90.274, "p95_ms": 87.491}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.297, "count": 52, "max_ms": 100.448, "p95_ms": 97.97}, "read_interval": {"avg_ms": 489.902, "count": 51, "max_ms": 19724.314, "p95_ms": 115.281}, "triton_wait": {"avg_ms": 80.262, "count": 52, "max_ms": 90.286, "p95_ms": 87.505}, "visualize": {"avg_ms": 3.521, "count": 52, "max_ms": 4.601, "p95_ms": 3.871}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2532}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4135.324, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5592b9f61e80] error while decoding MB 25 47, bytestream -5 [in#0/rtsp @ 0x5592b9f1be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5592b9f1be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:56:17+09:00", "last_error_summary": "[h264 @ 0x5592b9f61e80] error while decoding MB 25 47, bytestream -5 [in#0/rtsp @ 0x5592b9f1be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5592b9f1be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:43:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 751.575, "last_started_at": "2026-07-01T23:43:45+09:00", "pid": 30209, "process": {"exists": false, "pid": 30209}, "restart_count": 43, "restart_count_per_hour": 11, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.566, "sample_count": 2776, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 303897, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:56:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-01T23:56:17+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-01T23:56:17+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-01T23:56:17.340052+09:00", "mtime_epoch": 1782917777.3400524, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:56:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1011.75, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 32, "pushed_frames": 32, "read_frames": 32, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.067, "pushed": 1.067, "read": 1.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.509, "count": 31, "max_ms": 5.033, "p95_ms": 4.736}, "gstreamer_push": {"avg_ms": 1.564, "count": 32, "max_ms": 8.195, "p95_ms": 1.535}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.147, "count": 31, "max_ms": 74.294, "p95_ms": 73.84}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.581, "count": 31, "max_ms": 82.909, "p95_ms": 81.346}, "yolo11_640_fire_11class": {"avg_ms": 55.792, "count": 31, "max_ms": 76.69, "p95_ms": 75.075}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.522, "count": 31, "max_ms": 86.612, "p95_ms": 85.948}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.273, "count": 32, "max_ms": 98.69, "p95_ms": 97.928}, "read_interval": {"avg_ms": 938.564, "count": 32, "max_ms": 16503.689, "p95_ms": 10844.606}, "triton_wait": {"avg_ms": 59.911, "count": 31, "max_ms": 86.625, "p95_ms": 85.959}, "visualize": {"avg_ms": 3.759, "count": 31, "max_ms": 5.825, "p95_ms": 4.302}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2532}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4134.797, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-01T23:56:17+09:00", "last_error_summary": "[h264 @ 0x5592b9f61e80] error while decoding MB 25 47, bytestream -5 [in#0/rtsp @ 0x5592b9f1be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5592b9f1be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:56:17+09:00", "last_error_summary": "[h264 @ 0x5592b9f61e80] error while decoding MB 25 47, bytestream -5 [in#0/rtsp @ 0x5592b9f1be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5592b9f1be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:56:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 751.613, "last_started_at": "2026-07-01T23:43:45+09:00", "pid": 30209, "process": {"exists": false, "pid": 30209}, "restart_count": 43, "restart_count_per_hour": 11, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.566, "sample_count": 2777, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 303898, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:56:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-01T23:56:17+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-01T23:56:17+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-01T23:56:17.340052+09:00", "mtime_epoch": 1782917777.3400524, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:56:17+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1011.75, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 32, "pushed_frames": 32, "read_frames": 31, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.068, "pushed": 1.068, "read": 1.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.503, "count": 32, "max_ms": 5.033, "p95_ms": 4.736}, "gstreamer_push": {"avg_ms": 1.564, "count": 32, "max_ms": 8.195, "p95_ms": 1.535}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.742, "count": 32, "max_ms": 74.294, "p95_ms": 73.84}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.239, "count": 32, "max_ms": 82.909, "p95_ms": 81.346}, "yolo11_640_fire_11class": {"avg_ms": 56.243, "count": 32, "max_ms": 76.69, "p95_ms": 75.075}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.147, "count": 32, "max_ms": 86.612, "p95_ms": 85.948}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.098, "count": 32, "max_ms": 98.69, "p95_ms": 97.928}, "read_interval": {"avg_ms": 965.037, "count": 31, "max_ms": 16503.689, "p95_ms": 10844.606}, "triton_wait": {"avg_ms": 60.529, "count": 32, "max_ms": 86.625, "p95_ms": 85.959}, "visualize": {"avg_ms": 3.76, "count": 32, "max_ms": 5.825, "p95_ms": 4.302}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2532}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4134.688, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:56:17+09:00", "last_error_summary": "[h264 @ 0x5592b9f61e80] error while decoding MB 25 47, bytestream -5 [in#0/rtsp @ 0x5592b9f1be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5592b9f1be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:56:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 751.621, "last_started_at": "2026-07-01T23:43:45+09:00", "pid": 30209, "process": {"exists": false, "pid": 30209}, "restart_count": 43, "restart_count_per_hour": 11, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.566, "sample_count": 2776, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 303898, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:56:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-01T23:56:17+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-01T23:56:17+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-01T23:56:17.340052+09:00", "mtime_epoch": 1782917777.3400524, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:56:17+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1011.75, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 32, "pushed_frames": 32, "read_frames": 31, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.068, "pushed": 1.068, "read": 1.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.503, "count": 32, "max_ms": 5.033, "p95_ms": 4.736}, "gstreamer_push": {"avg_ms": 1.564, "count": 32, "max_ms": 8.195, "p95_ms": 1.535}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.742, "count": 32, "max_ms": 74.294, "p95_ms": 73.84}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.239, "count": 32, "max_ms": 82.909, "p95_ms": 81.346}, "yolo11_640_fire_11class": {"avg_ms": 56.243, "count": 32, "max_ms": 76.69, "p95_ms": 75.075}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.147, "count": 32, "max_ms": 86.612, "p95_ms": 85.948}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.098, "count": 32, "max_ms": 98.69, "p95_ms": 97.928}, "read_interval": {"avg_ms": 965.037, "count": 31, "max_ms": 16503.689, "p95_ms": 10844.606}, "triton_wait": {"avg_ms": 60.529, "count": 32, "max_ms": 86.625, "p95_ms": 85.959}, "visualize": {"avg_ms": 3.76, "count": 32, "max_ms": 5.825, "p95_ms": 4.302}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 2532}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4170.793, "limit_mb": 8796093022207.996}, "detail": {"pid": 30468, "restart_count": 44, "start_count": 45}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:56:17+09:00", "last_error_summary": "[h264 @ 0x5592b9f61e80] error while decoding MB 25 47, bytestream -5 [in#0/rtsp @ 0x5592b9f1be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5592b9f1be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:56:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-01T23:56:19+09:00", "pid": 30468, "process": {"exists": true, "fd_count": 5, "hwm_mb": 73.898, "name": "ffmpeg", "pid": 30468, "rss_mb": 73.898, "state": "S (sleeping)", "threads": 19}, "restart_count": 44, "restart_count_per_hour": 12, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.485, "sample_count": 2739, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 303898, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-01T23:56:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.234, "last_frame_pushed_at": "2026-07-01T23:56:17+09:00", "last_frame_read_age_sec": 4.328, "last_frame_read_at": "2026-07-01T23:56:17+09:00", "mjpeg": {"age_sec": 4.22, "exists": true, "mtime": "2026-07-01T23:56:17.452055+09:00", "mtime_epoch": 1782917777.4520547, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 93535}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-01T23:56:21+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1011.75, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.639, "pushed": 0.639, "read": 0.639}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.483, "count": 14, "max_ms": 4.736, "p95_ms": 4.736}, "gstreamer_push": {"avg_ms": 1.849, "count": 14, "max_ms": 8.195, "p95_ms": 8.195}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.637, "count": 14, "max_ms": 67.354, "p95_ms": 67.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.593, "count": 14, "max_ms": 82.909, "p95_ms": 82.909}, "yolo11_640_fire_11class": {"avg_ms": 58.341, "count": 14, "max_ms": 75.075, "p95_ms": 75.075}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.854, "count": 14, "max_ms": 86.612, "p95_ms": 86.612}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.583, "count": 14, "max_ms": 98.69, "p95_ms": 98.69}, "read_interval": {"avg_ms": 2030.554, "count": 14, "max_ms": 16503.689, "p95_ms": 16503.689}, "triton_wait": {"avg_ms": 63.07, "count": 14, "max_ms": 86.625, "p95_ms": 86.625}, "visualize": {"avg_ms": 3.669, "count": 14, "max_ms": 4.042, "p95_ms": 4.042}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.911, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}